Allow to specify extra esp-idf components in manifest metadata

Makes it possible to specify extra components in the `package.metadata.esp-idf-sys.extra_components`
array in the root crate's manifest as well as all direct dependencies.
This change also instructs cmake to build all extra components, but without generating any bindings.

build/build.rs

@@ -1,9 +1,7 @@
 #[cfg(not(any(feature = "pio", feature = "native")))]
 compile_error!("One of the features `pio` or `native` must be selected.");
 
-use std::env;
 use std::iter::once;
-use std::path::PathBuf;
 
 use ::bindgen::callbacks::{IntKind, ParseCallbacks};
 use anyhow::*;
@@ -80,10 +78,10 @@ fn main() -> anyhow::Result<()> {
         )
     })?;
 
-    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
+    let manifest_dir = manifest_dir()?;
 
     let header_file = path_buf![
-        manifest_dir,
+        &manifest_dir,
         "src",
         "include",
         if mcu == "esp8266" {

build/common.rs

@@ -1,4 +1,5 @@
 use std::collections::HashSet;
+use std::ffi::OsStr;
 use std::fmt::Display;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
@@ -340,3 +341,39 @@ impl Display for InstallDir {
 pub fn workspace_dir() -> Result<PathBuf> {
     cargo::workspace_dir().ok_or_else(|| anyhow!("Cannot fetch crate's workspace dir"))
 }
+
+pub fn manifest_dir() -> Result<PathBuf> {
+    std::env::var_os("CARGO_MANIFEST_DIR")
+        .ok_or_else(|| {
+            anyhow!(
+                "Environment variable `CARGO_MANIFEST_DIR` unavailable: not in cargo build script"
+            )
+        })
+        .map(PathBuf::from)
+}
+
+/// Create a cmake list (`;`-separated strings), escape all `;` and on Windows make sure
+/// paths don't contain `\`.
+pub fn to_cmake_path_list(iter: impl IntoIterator<Item = impl AsRef<OsStr>>) -> Result<String> {
+    let mut accu = String::new();
+    for p in iter {
+        let p: &str = p.as_ref().try_to_str()?;
+        if !accu.is_empty() {
+            accu.push(';');
+        }
+
+        // Escape all `;` since cmake uses them as separators.
+        let p = p.replace(';', "\\;");
+
+        accu.push_str(
+            // Windows uses `\` as directory separators which cmake can't deal with, so we
+            // convert all back-slashes to forward-slashes here.
+            &if cfg!(windows) {
+                p.replace('\\', "/")
+            } else {
+                p
+            },
+        );
+    }
+    Ok(accu)
+}

build/config.rs

@@ -1,8 +1,7 @@
 use std::path::PathBuf;
 
 use anyhow::{anyhow, bail, Context, Result};
-use embuild::cargo;
-use serde::Deserialize;
+use serde::{Deserialize, Deserializer};
 
 use crate::common::{workspace_dir, InstallDir, InstallDirLocation};
 
@@ -47,21 +46,13 @@ impl BuildConfig {
     /// Note: The environment variables to deserialize must be valid rust [`String`]s
     /// (can only contain utf-8).
     pub fn try_from_env() -> Result<BuildConfig> {
-        for var in utils::serde_introspect::<BuildConfig>() {
-            cargo::track_env_var(var.to_uppercase());
-        }
-
-        let cfg: BuildConfig = envy::from_env()?;
+        let cfg: BuildConfig = utils::parse_from_env(&[])?;
 
         #[cfg(feature = "native")]
         let cfg = {
             use crate::native::cargo_driver::config::NativeConfig;
-            for var in utils::serde_introspect::<NativeConfig>() {
-                cargo::track_env_var(var.to_uppercase());
-            }
-
             BuildConfig {
-                native: envy::from_env()?,
+                native: NativeConfig::try_from_env()?,
                 ..cfg
             }
         };
@@ -128,7 +119,7 @@ impl BuildConfig {
         };
 
         // Deserialize the options from the `esp-idf-sys` object.
-        let InnerMetadata {
+        let EspIdfSys {
             v:
                 BuildConfig {
                     esp_idf_tools_install_dir,
@@ -138,7 +129,7 @@ impl BuildConfig {
                     native: _,
                     esp_idf_sys_root_crate: _,
                 },
-        } = InnerMetadata::deserialize(&root_package.metadata)?;
+        } = EspIdfSys::deserialize(&root_package.metadata)?;
 
         // Update all options that are currently [`None`].
         utils::set_when_none(&mut self.esp_idf_sdkconfig, esp_idf_sdkconfig);
@@ -160,42 +151,64 @@ impl BuildConfig {
 }
 
 /// A container to defer to the `esp-idf-sys` table of the metadata.
-#[derive(Deserialize)]
-pub struct InnerMetadata<T: Default> {
+#[derive(Deserialize, Default)]
+pub struct EspIdfSys<T: Default> {
     #[serde(default, rename = "esp-idf-sys")]
     pub v: T,
 }
 
+impl<'a, T> EspIdfSys<T>
+where
+    T: Default,
+    T: Deserialize<'a>,
+{
+    /// Deserialize an `esp-idf-sys` field.
+    pub fn deserialize<D>(de: D) -> Result<Self>
+    where
+        D: Deserializer<'a>,
+        D::Error: Send + Sync + std::error::Error + 'static,
+    {
+        let result = Option::<Self>::deserialize(de)
+            .with_context(|| anyhow!("could not read build config from manifest metadata"))?;
+        Ok(result.unwrap_or_default())
+    }
+}
+
 mod parse {
     use std::path::PathBuf;
 
     use serde::{Deserialize, Deserializer};
 
+    use super::utils::ValueOrVec;
+
     pub fn sdkconfig_defaults<'d, D: Deserializer<'d>>(
         de: D,
     ) -> Result<Option<Vec<PathBuf>>, D::Error> {
-        #[derive(Deserialize)]
-        #[serde(untagged)]
-        enum StringOrVec {
-            Str(String),
-            Vec(Vec<PathBuf>),
-        }
-
-        Option::<StringOrVec>::deserialize(de).map(|val| match val {
-            Some(StringOrVec::Str(s)) => Some(
+        Option::<ValueOrVec<String, PathBuf>>::deserialize(de).map(|val| match val {
+            Some(ValueOrVec::Val(s)) => Some(
                 s.split(';')
                     .filter(|s| !s.is_empty())
                     .map(PathBuf::from)
                     .collect(),
             ),
-            Some(StringOrVec::Vec(v)) => Some(v),
+            Some(ValueOrVec::Vec(v)) => Some(v),
             None => None,
         })
     }
 }
 
 pub mod utils {
-    use serde::de::{self, Deserialize, Deserializer, Visitor};
+    use embuild::cargo;
+    use serde::de::{self, Deserializer, Visitor};
+    use serde::Deserialize;
+
+    /// A helper enum for deserializing a single value or a list of values.
+    #[derive(Deserialize, Debug)]
+    #[serde(untagged)]
+    pub enum ValueOrVec<V, E = V> {
+        Val(V),
+        Vec(Vec<E>),
+    }
 
     /// Gets the serialization names for structs.
     ///
@@ -253,4 +266,23 @@ pub mod utils {
             *val = new;
         }
     }
+
+    /// Parse the value from the environment variables and exclude all fields of `T` that
+    /// are in `exclude_list`.
+    pub fn parse_from_env<T>(exclude_list: &[&str]) -> envy::Result<T>
+    where
+        T: for<'d> Deserialize<'d>,
+    {
+        let var_filter = |k: &str| !exclude_list.contains(&k);
+
+        for var in serde_introspect::<T>()
+            .into_iter()
+            .filter(|s| var_filter(s))
+        {
+            cargo::track_env_var(var.to_uppercase());
+        }
+
+        let vars = std::env::vars().filter(|(key, _)| var_filter(&key.to_lowercase()));
+        envy::from_iter(vars)
+    }
 }

build/native/cargo_driver.rs

@@ -1,6 +1,5 @@
 use std::convert::TryFrom;
-use std::ffi::OsString;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::str::FromStr;
 use std::{env, fs};
 
@@ -16,13 +15,8 @@ use embuild::{bindgen, build, cargo, cmake, espidf, git, kconfig, path_buf};
 
 use self::chip::Chip;
 use crate::common::{
-    self,
-    list_specific_sdkconfigs,
-    workspace_dir,
-    EspIdfBuildOutput,
-    EspIdfComponents,
-    InstallDir,
-    V_4_3_2_PATCHES,
+    self, list_specific_sdkconfigs, manifest_dir, to_cmake_path_list, workspace_dir,
+    EspIdfBuildOutput, EspIdfComponents, InstallDir, V_4_3_2_PATCHES,
 };
 use crate::config::{BuildConfig, ESP_IDF_GLOB_VAR_PREFIX, ESP_IDF_TOOLS_INSTALL_DIR_VAR};
 
@@ -33,16 +27,12 @@ pub fn build() -> Result<EspIdfBuildOutput> {
     let out_dir = cargo::out_dir();
     let target = env::var("TARGET")?;
     let workspace_dir = workspace_dir()?;
-    let manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?);
+    let manifest_dir = manifest_dir()?;
 
     let config = BuildConfig::try_from_env().map(|mut config| {
-        config
-            .with_cargo_metadata()
-            .context("failed to read configuration from manifest metadata")
-            .into_warning();
+        config.with_cargo_metadata().into_warning();
         config
     })?;
-
     config.print();
 
     let chip = if let Some(mcu) = config.mcu.clone() {
@@ -52,9 +42,9 @@ pub fn build() -> Result<EspIdfBuildOutput> {
     };
     let chip_name = chip.to_string();
     let profile = common::build_profile();
-
     let cmake_generator = config.native.esp_idf_cmake_generator();
 
+    // A closure to specify which tools `idf-tools.py` should install.
     let make_tools = move |repo: &git::Repository,
                            version: &Result<espidf::EspIdfVersion>|
           -> Result<Vec<espidf::Tools>> {
@@ -93,6 +83,7 @@ pub fn build() -> Result<EspIdfBuildOutput> {
     let require_from_env = install_dir.is_from_env();
     let maybe_from_env = require_from_env || allow_from_env;
 
+    // Closure to install the esp-idf using `embuild::espidf::Installer`.
     let install = |esp_idf_origin: EspIdfOrigin| -> Result<espidf::EspIdf> {
         match &esp_idf_origin {
             EspIdfOrigin::Custom(repo) => {
@@ -190,6 +181,10 @@ pub fn build() -> Result<EspIdfBuildOutput> {
 
     env::set_var("PATH", &idf.exported_path);
 
+    // The `kconfig.cmake` script looks at this variable if it should compile `mconf` on windows.
+    // But this variable is also present when using git-bash which doesn't have gcc.
+    env::remove_var("MSYSTEM");
+
     // Remove the sdkconfig file generated by the esp-idf so that potential changes
     // in the user provided sdkconfig and sdkconfig.defaults don't get ignored.
     // TODO: I'm really not sure why we have to do this.
@@ -214,12 +209,8 @@ pub fn build() -> Result<EspIdfBuildOutput> {
         copy_file_if_different(&file.0, &out_dir)?;
     }
 
-    // The `kconfig.cmake` script looks at this variable if it should compile `mconf` on windows.
-    // But this variable is also present when using git-bash which doesn't have gcc.
-    env::remove_var("MSYSTEM");
-
-    // Resolve `ESP_IDF_SDKCONFIG` and `ESP_IDF_SDKCONFIG_DEFAULTS` to an absolute path
-    // relative to the workspace directory if not empty.
+    // Resolve the `sdkconfig` and all `sdkconfig.defaults` files specified in the build
+    // config.
     let sdkconfig = {
         let file = config.esp_idf_sdkconfig();
         let path = Path::new(&file).abspath_relative_to(&workspace_dir);
@@ -255,26 +246,14 @@ pub fn build() -> Result<EspIdfBuildOutput> {
         result
     };
 
-    let defaults_files = sdkconfig_defaults
-        .iter()
-        // Use the `sdkconfig` as a defaults file to prevent it from being changed by the
-        // build. It must be the last defaults file so that its options have precendence
-        // over any actual defaults from files before it.
-        .chain(sdkconfig.as_ref())
-        .try_fold(OsString::new(), |mut accu, p| -> Result<OsString> {
-            if !accu.is_empty() {
-                accu.push(";");
-            }
-            // Windows uses `\` as directory separators which cmake can't deal with, so we
-            // convert all back-slashes to forward-slashes here. This would be tedious to
-            // do with an `OsString` so we have to convert it to `str` first.
-            if cfg!(windows) {
-                accu.push(p.try_to_str()?.replace('\\', "/"));
-            } else {
-                accu.push(p);
-            }
-            Ok(accu)
-        })?;
+    let defaults_files = to_cmake_path_list(
+        sdkconfig_defaults
+            .iter()
+            // Use the `sdkconfig` as a defaults file to prevent it from being changed by the
+            // build. It must be the last defaults file so that its options have precendence
+            // over any actual defaults from files before it.
+            .chain(sdkconfig.as_ref()),
+    )?;
 
     let cmake_toolchain_file = path_buf![
         &idf.repository.worktree(),
@@ -304,6 +283,9 @@ pub fn build() -> Result<EspIdfBuildOutput> {
         )
     };
 
+    // Get the directories of all extra components to build.
+    let extra_component_dirs = to_cmake_path_list(config.native.extra_component_dirs()?)?;
+
     // `cmake::Config` automatically uses `<out_dir>/build` and there is no way to query
     // what build directory it sets, so we hard-code it.
     let cmake_build_dir = out_dir.join("build");
@@ -326,6 +308,7 @@ pub fn build() -> Result<EspIdfBuildOutput> {
         .asmflag(asm_flags)
         .cflag(c_flags)
         .cxxflag(cxx_flags)
+        .env("EXTRA_COMPONENT_DIRS", extra_component_dirs)
         .env("IDF_PATH", &idf.repository.worktree())
         .env("PATH", &idf.exported_path)
         .env("SDKCONFIG_DEFAULTS", defaults_files)