Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[core] Split configuration for device and others in compile model #26977

Merged
Merged
Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
e10f512
Split configuration for device and others
praasz Oct 9, 2024
32750d6
Fix call compile model with weights
praasz Oct 9, 2024
cdc5655
Remove modify core state in compile_model
praasz Oct 13, 2024
f9a2362
Remove not used variable
praasz Oct 13, 2024
b1387b2
Fix MSVC build error
praasz Oct 13, 2024
4f33336
Auto-Batch remove prop validation to meta device
praasz Oct 14, 2024
a6ad92e
Add missing supported properties in Template
praasz Oct 14, 2024
ce4a5a0
Add parse only properties in auto-batch
praasz Oct 14, 2024
d9e02b9
Merge remote-tracking branch 'origin/master' into split-config-in-com…
praasz Oct 14, 2024
945aab2
Template plugin sync supported properties
praasz Oct 14, 2024
a89eed8
CPU plugin sync supported properties
praasz Oct 14, 2024
0426bda
Hetero plugin sync supported properties
praasz Oct 14, 2024
8ae4d36
Make `parse_only_meta_device` member private
praasz Oct 14, 2024
1220a36
Merge branch 'master' into split-config-in-compile-model
praasz Oct 16, 2024
2a9b1e6
Merge branch 'master' into split-config-in-compile-model
praasz Oct 21, 2024
327c93a
Remove not used parameter in function
praasz Oct 22, 2024
82e337f
Merge branch 'master' into split-config-in-compile-model
praasz Oct 22, 2024
45e270d
Merge branch 'master' into split-config-in-compile-model
praasz Oct 23, 2024
ec5b0cf
Merge branch 'master' into split-config-in-compile-model
praasz Oct 25, 2024
837f633
Merge branch 'master' into split-config-in-compile-model
praasz Oct 28, 2024
cd5516b
Merge branch 'master' into split-config-in-compile-model
praasz Nov 12, 2024
803b5c3
Merge branch 'master' into split-config-in-compile-model
praasz Nov 14, 2024
b7e1ea7
Merge branch 'master' into split-config-in-compile-model
praasz Nov 15, 2024
d9dfae2
Merge branch 'master' into split-config-in-compile-model
praasz Nov 18, 2024
fabdba5
Merge branch 'master' into split-config-in-compile-model
praasz Nov 18, 2024
82b6c5c
Add kv_cache_precision as supported property by GPU
praasz Nov 20, 2024
982b9e8
Merge branch 'master' into split-config-in-compile-model
praasz Nov 21, 2024
c1ffd36
Merge branch 'master' into split-config-in-compile-model
praasz Nov 29, 2024
ce822f7
Merge branch 'master' into split-config-in-compile-model
praasz Dec 2, 2024
2772f3d
Filter core properties by using local core config
praasz Dec 10, 2024
46c05ba
Revert changes
praasz Dec 10, 2024
434b67f
Update `parseDeviceNameIntoConfig` to provide local core configuration
praasz Dec 11, 2024
78b5226
Merge remote-tracking branch 'origin/master' into split-config-in-com…
praasz Dec 11, 2024
c0b700b
Fix issues after merge
praasz Dec 11, 2024
7dc7b73
Clean core properties for HW devices in parseDeviceNameIntoConfig
praasz Dec 11, 2024
9c7f13a
Add comment about cache_dir removal from compile config
praasz Dec 11, 2024
511bb88
Re order code in core_impl.cpp
praasz Dec 11, 2024
ca0ba63
Merge branch 'master' into split-config-in-compile-model
praasz Dec 11, 2024
99508e5
Merge remote-tracking branch 'origin/master' into split-config-in-com…
praasz Dec 12, 2024
f182198
Explicitly remove copy assign operator for CoreConfig
praasz Dec 12, 2024
3370ef8
Merge branch 'master' into split-config-in-compile-model
praasz Dec 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 82 additions & 47 deletions src/inference/src/dev/core_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,8 @@ void clean_batch_properties(const std::string& deviceName, ov::AnyMap& config, c
}
}
}

constexpr bool rw_only = true;
} // namespace

bool ov::is_config_applicable(const std::string& user_device_name, const std::string& subprop_device_name) {
Expand Down Expand Up @@ -544,8 +546,8 @@ void ov::CoreImpl::register_plugins_in_registry(const std::string& xml_config_fi
}
}

ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const {
OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin");
ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName, bool on_create_filter_config) const {
OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin::supported_config");

auto deviceName = pluginName;
if (deviceName == ov::DEFAULT_DEVICE_NAME)
Expand Down Expand Up @@ -690,13 +692,17 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const {
ov::DeviceIDParser parser(pluginDesc.first);
if (pluginDesc.first.find(deviceName) != std::string::npos && !parser.get_device_id().empty()) {
pluginDesc.second.defaultConfig[deviceKey] = parser.get_device_id();
plugin.set_property(pluginDesc.second.defaultConfig);
plugin.set_property(
on_create_filter_config
? get_hw_plugin_properties_or_forward(plugin, pluginDesc.second.defaultConfig)
: pluginDesc.second.defaultConfig);
}
}
}

// set global device-id independent settings to plugin
plugin.set_property(desc.defaultConfig);
plugin.set_property(on_create_filter_config
? get_hw_plugin_properties_or_forward(plugin, desc.defaultConfig)
: desc.defaultConfig);
});
}

Expand Down Expand Up @@ -728,6 +734,12 @@ ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const {
}
}

ov::Plugin ov::CoreImpl::get_plugin(const std::string& pluginName) const {
OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "CoreImpl::get_plugin");
constexpr auto on_plugin_create_filter_config = false;
return get_plugin(pluginName, on_plugin_create_filter_config);
}

ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<const ov::Model>& model_,
const std::string& device_name,
const ov::AnyMap& config) const {
Expand All @@ -738,7 +750,7 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<
auto model = apply_auto_batching(model_, deviceName, config_with_batch);

auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(device_name));
auto plugin = get_plugin(parsed._deviceName);
auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName));
ov::SoPtr<ov::ICompiledModel> res;
auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager;
// Skip caching for proxy plugin. HW plugin will load network from the cache
Expand All @@ -749,12 +761,12 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<
res = load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr<ov::IRemoteContext>{}, [&]() {
return compile_model_and_cache(plugin,
model,
parsed._config,
get_hw_plugin_properties_or_forward(plugin, parsed._config),
ov::SoPtr<ov::IRemoteContext>{},
cacheContent);
});
} else {
res = plugin.compile_model(model, parsed._config);
res = plugin.compile_model(model, get_hw_plugin_properties_or_forward(plugin, parsed._config));
}
return res;
}
Expand All @@ -771,7 +783,7 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<
auto model = apply_auto_batching(model_, deviceName, config_with_batch);

auto parsed = parseDeviceNameIntoConfig(deviceName, config_with_batch, is_proxy_device(deviceName));
auto plugin = get_plugin(parsed._deviceName);
auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName));
ov::SoPtr<ov::ICompiledModel> res;
auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager;
// Skip caching for proxy plugin. HW plugin will load network from the cache
Expand All @@ -780,10 +792,14 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::shared_ptr<
cacheContent.blobId = ov::ModelCache::compute_hash(model, create_compile_config(plugin, parsed._config));
std::unique_ptr<CacheGuardEntry> lock = cacheGuard.get_hash_lock(cacheContent.blobId);
res = load_model_from_cache(cacheContent, plugin, parsed._config, context, [&]() {
return compile_model_and_cache(plugin, model, parsed._config, context, cacheContent);
return compile_model_and_cache(plugin,
model,
get_hw_plugin_properties_or_forward(plugin, parsed._config),
context,
cacheContent);
});
} else {
res = plugin.compile_model(model, context, parsed._config);
res = plugin.compile_model(model, context, get_hw_plugin_properties_or_forward(plugin, parsed._config));
}
return res;
}
Expand All @@ -794,11 +810,10 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::string& mod
OV_ITT_SCOPE(FIRST_INFERENCE, ov::itt::domains::LoadTime, "Core::compile_model::Path");
auto parsed = parseDeviceNameIntoConfig(device_name, config);
// in case of compile_model(file_name), we need to clear-up core-level properties
auto plugin = get_plugin(parsed._deviceName);
auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName));
ov::SoPtr<ov::ICompiledModel> compiled_model;

auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager;

if (cacheManager && device_supports_model_caching(plugin) && !is_proxy_device(plugin)) {
// Skip caching for proxy plugin. HW plugin will load network from the cache
CacheContent cacheContent{cacheManager, model_path};
Expand All @@ -807,10 +822,14 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::string& mod
compiled_model =
load_model_from_cache(cacheContent, plugin, parsed._config, ov::SoPtr<ov::IRemoteContext>{}, [&]() {
auto model = read_model(model_path, std::string{});
return compile_model_and_cache(plugin, model, parsed._config, {}, cacheContent);
return compile_model_and_cache(plugin,
model,
get_hw_plugin_properties_or_forward(plugin, parsed._config),
{},
cacheContent);
});
} else {
compiled_model = plugin.compile_model(model_path, parsed._config);
compiled_model = plugin.compile_model(model_path, get_hw_plugin_properties_or_forward(plugin, parsed._config));
}
return compiled_model;
}
Expand All @@ -822,7 +841,7 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::string& mod
OV_ITT_SCOPED_TASK(ov::itt::domains::OV, "Core::compile_model::from_memory");
auto parsed = parseDeviceNameIntoConfig(device_name, config);
// in case of compile_model(file_name), we need to clear-up core-level properties
auto plugin = get_plugin(parsed._deviceName);
auto plugin = get_plugin(parsed._deviceName, !is_virtual_device(parsed._deviceName));
ov::SoPtr<ov::ICompiledModel> compiled_model;

auto cacheManager = coreConfig.get_cache_config_for_device(plugin, parsed._config)._cacheManager;
Expand All @@ -837,13 +856,13 @@ ov::SoPtr<ov::ICompiledModel> ov::CoreImpl::compile_model(const std::string& mod
auto model = read_model(model_str, weights);
return compile_model_and_cache(plugin,
model,
parsed._config,
get_hw_plugin_properties_or_forward(plugin, parsed._config),
ov::SoPtr<ov::IRemoteContext>{},
cacheContent);
});
} else {
auto model = read_model(model_str, weights);
compiled_model = plugin.compile_model(model, parsed._config);
compiled_model = plugin.compile_model(model, get_hw_plugin_properties_or_forward(plugin, parsed._config));
}
return compiled_model;
}
Expand Down Expand Up @@ -934,28 +953,11 @@ ov::SoPtr<ov::IRemoteContext> ov::CoreImpl::create_context(const std::string& de
return get_plugin(parsed._deviceName).create_context(parsed._config);
}

ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name,
const ov::AnyMap& user_properties,
const bool keep_core_property) const {
if (is_virtual_device(full_device_name)) {
// Considerations:
// 1. in case of virtual devices all the magic will happen on the level when
// virtual device calls ICore::get_supported_property for real HW devices
// so, for now we can return user properties almost as is without any
// filtering / flattening
// 2. The only exception here: while common properties like ov::num::streams or
// ov::hint::performance_mode are shared across all the devices, the
// ov::device::priority cannot be shared, because it's specific for current virtual
// plugin. So, we need to remove ov::device::priorities from the list, because it's
// supposed to be set for current virtual plugin and cannot be propagated down
ov::AnyMap return_properties = user_properties;
auto device_priorities_it = return_properties.find(ov::device::priorities.name());
if (device_priorities_it != return_properties.end()) {
return_properties.erase(device_priorities_it);
}

return return_properties;
}
ov::AnyMap ov::CoreImpl::get_supported_property(const Plugin& plugin,
const ov::AnyMap& config,
bool keep_core,
bool rw_only) const {
const auto& full_device_name = plugin.get_name();

static const std::vector<std::string> core_level_properties = {
ov::cache_dir.name(),
Expand All @@ -965,21 +967,20 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n
ov::hint::allow_auto_batching.name(),
};

const auto flattened = ov::parseDeviceNameIntoConfig(full_device_name, user_properties, true);
const std::string& device_name = flattened._deviceName;
const auto flattened = ov::parseDeviceNameIntoConfig(full_device_name, config, true);
const auto& flattened_config = flattened._config;

// virtual plugins should bypass core-level properties to HW plugins
// so, we need to report them as supported
std::vector<std::string> supported_config_keys;
if (keep_core_property) {
if (keep_core) {
supported_config_keys = core_level_properties;
}

// try to search against OV API 2.0' mutable supported_properties
try {
for (auto&& property : ICore::get_property(device_name, ov::supported_properties, {})) {
if (property.is_mutable()) {
for (auto&& property : plugin.get_property(ov::supported_properties)) {
if (!rw_only || property.is_mutable()) {
supported_config_keys.emplace_back(std::move(property));
}
}
Expand All @@ -988,8 +989,8 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n

// try to search against internal supported_properties
try {
for (auto&& property : ICore::get_property(device_name, ov::internal::supported_properties, {})) {
if (property.is_mutable()) {
for (auto&& property : plugin.get_property(ov::internal::supported_properties)) {
if (!rw_only || property.is_mutable()) {
supported_config_keys.emplace_back(std::move(property));
}
}
Expand All @@ -1007,6 +1008,40 @@ ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_n
return supported_config;
}

ov::AnyMap ov::CoreImpl::get_supported_property(const std::string& full_device_name,
const ov::AnyMap& user_properties,
const bool keep_core_property) const {
if (is_virtual_device(full_device_name)) {
// Considerations:
// 1. in case of virtual devices all the magic will happen on the level when
// virtual device calls ICore::get_supported_property for real HW devices
// so, for now we can return user properties almost as is without any
// filtering / flattening
// 2. The only exception here: while common properties like ov::num::streams or
// ov::hint::performance_mode are shared across all the devices, the
// ov::device::priority cannot be shared, because it's specific for current virtual
// plugin. So, we need to remove ov::device::priorities from the list, because it's
// supposed to be set for current virtual plugin and cannot be propagated down
auto return_properties = user_properties;
auto device_priorities_it = return_properties.find(ov::device::priorities.name());
if (device_priorities_it != return_properties.end()) {
return_properties.erase(device_priorities_it);
}
return return_properties;
} else {
const auto parsed = ov::parseDeviceNameIntoConfig(full_device_name);
return get_supported_property(get_plugin(parsed._deviceName), user_properties, keep_core_property, rw_only);
}
}

ov::AnyMap ov::CoreImpl::get_hw_plugin_properties_or_forward(const Plugin& plugin, const AnyMap& config) const {
constexpr auto keep_core = false;
const auto& device_name = plugin.get_name();
const auto forward_config = is_virtual_device(device_name) || is_proxy_device(device_name);

return forward_config ? config : get_supported_property(plugin, config, keep_core, !rw_only);
praasz marked this conversation as resolved.
Show resolved Hide resolved
}

ov::SoPtr<ov::IRemoteContext> ov::CoreImpl::get_default_context(const std::string& device_name) const {
auto parsed = ov::parseDeviceNameIntoConfig(device_name);
return get_plugin(parsed._deviceName).get_default_context(parsed._config);
Expand Down
10 changes: 9 additions & 1 deletion src/inference/src/dev/core_impl.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,12 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this<ov::ICore
}
void add_extensions_unsafe(const std::vector<ov::Extension::Ptr>& extensions) const;

Plugin get_plugin(const std::string& pluginName, bool on_create_filter_config) const;

AnyMap get_supported_property(const Plugin& plugin, const AnyMap& config, bool keep_core, bool rw_only) const;

AnyMap get_hw_plugin_properties_or_forward(const Plugin& plugin, const AnyMap& config) const;

public:
CoreImpl();

Expand Down Expand Up @@ -291,7 +297,9 @@ class CoreImpl : public ov::ICore, public std::enable_shared_from_this<ov::ICore

ov::SoPtr<ov::IRemoteContext> create_context(const std::string& device_name, const AnyMap& args) const override;

ov::AnyMap get_supported_property(const std::string& device_name, const ov::AnyMap& config, const bool keep_core_property = true) const override;
ov::AnyMap get_supported_property(const std::string& device_name,
const ov::AnyMap& config,
const bool keep_core_property = true) const override;

ov::SoPtr<ov::IRemoteContext> get_default_context(const std::string& device_name) const override;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -190,4 +190,4 @@ TEST_F(AutoFuncTests, compiled_with_cache_enabled_batch_enabled) {
ASSERT_EQ(ov::test::utils::listFilesWithExt(cache_path, "blob").size(), 5);
core.set_property(ov::cache_dir(""));
#endif
}
}
12 changes: 9 additions & 3 deletions src/plugins/auto_batch/src/plugin.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,16 @@ DeviceInformation Plugin::parse_batch_device(const std::string& device_with_batc
return {std::move(deviceName), {{}}, static_cast<uint32_t>(batch)};
}

DeviceInformation Plugin::parse_meta_device(const std::string& devices_batch_config,
const ov::AnyMap& user_config) const {
DeviceInformation Plugin::parse_only_meta_device(const std::string& devices_batch_config,
const ov::AnyMap& user_config) const {
auto meta_device = parse_batch_device(devices_batch_config);
meta_device.device_config = get_core()->get_supported_property(meta_device.device_name, user_config);
return meta_device;
}

DeviceInformation Plugin::parse_meta_device(const std::string& devices_batch_config,
const ov::AnyMap& user_config) const {
auto meta_device = parse_only_meta_device(devices_batch_config, user_config);
// check that no irrelevant config-keys left
for (const auto& k : user_config) {
const auto& name = k.first;
Expand Down Expand Up @@ -140,7 +146,7 @@ std::shared_ptr<ov::ICompiledModel> Plugin::compile_model(const std::shared_ptr<
if (device_batch == full_properties.end()) {
OPENVINO_THROW("ov::device::priorities key for AUTO BATCH is not set for BATCH device");
}
auto meta_device = parse_meta_device(device_batch->second.as<std::string>(), properties);
auto meta_device = parse_only_meta_device(device_batch->second.as<std::string>(), properties);

const auto& device_name = meta_device.device_name;
const auto& device_config = meta_device.device_config;
Expand Down
5 changes: 4 additions & 1 deletion src/plugins/auto_batch/src/plugin.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,10 @@ class Plugin : public ov::IPlugin {
static DeviceInformation parse_batch_device(const std::string& device_with_batch);

private:
DeviceInformation parse_only_meta_device(const std::string& devices_batch_config,
const ov::AnyMap& user_config) const;

mutable ov::AnyMap m_plugin_config;
};
} // namespace autobatch_plugin
} // namespace ov
} // namespace ov
5 changes: 4 additions & 1 deletion src/plugins/hetero/src/plugin.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -283,7 +283,10 @@ ov::Any ov::hetero::Plugin::get_property(const std::string& name, const ov::AnyM
return decltype(ov::supported_properties)::value_type(std::move(supported_properties));
} else if (ov::internal::supported_properties == name) {
return decltype(ov::internal::supported_properties)::value_type{
ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO}};
ov::PropertyName{ov::internal::caching_properties.name(), ov::PropertyMutability::RO},
// write-only as internal
ov::PropertyName{ov::cache_encryption_callbacks.name(), ov::PropertyMutability::WO},
};
} else if (ov::device::full_name == name) {
return decltype(ov::device::full_name)::value_type{get_device_name()};
} else if (ov::internal::caching_properties == name) {
Expand Down
Loading
Loading