From 1e45fa84aeb4734cb412dc998bf07013790c58db Mon Sep 17 00:00:00 2001 From: bg-furiosa <143981183+bg-furiosa@users.noreply.github.com> Date: Tue, 23 Apr 2024 18:51:09 +0900 Subject: [PATCH] add hardware topology module (#105) * add hwloc wrapper * add topology module * add show_topology example * update build.rs to build and link hwloc statically * generate hwloc bindings * add HardwareTopologyHint trait --- .github/workflows/rust.yml | 2 +- Cargo.lock | 387 ++++++- device-api/Cargo.toml | 22 +- device-api/bin/show_topology.rs | 39 + device-api/build.rs | 121 +++ device-api/src/device.rs | 10 +- device-api/src/error.rs | 8 + device-api/src/lib.rs | 1 + device-api/src/proc.rs | 1 + device-api/src/topology/hwloc.rs | 185 ++++ device-api/src/topology/hwloc_binding.rs | 1 + device-api/src/topology/mod.rs | 327 ++++++ device-api/src/topology/test.xml | 1252 ++++++++++++++++++++++ 13 files changed, 2332 insertions(+), 24 deletions(-) create mode 100644 device-api/bin/show_topology.rs create mode 100644 device-api/build.rs create mode 100644 device-api/src/topology/hwloc.rs create mode 100644 device-api/src/topology/hwloc_binding.rs create mode 100644 device-api/src/topology/mod.rs create mode 100644 device-api/src/topology/test.xml diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 1ff7017..25dfc7d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - name: Install components + - name: Install rust components run: rustup component add clippy rustfmt - name: lint run: cargo fmt --all --check && cargo -q clippy --all-targets --features blocking -- -D rust_2018_idioms -D warnings diff --git a/Cargo.lock b/Cargo.lock index 70a9992..2899139 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -53,10 +53,17 @@ dependencies = [ ] [[package]] -name = "array_tool" -version = "1.0.3" +name = "attohttpc" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f8cb5d814eb646a863c4f24978cff2880c4be96ad8cde2c0f0678732902e271" +checksum = "9a13149d0cf3f7f9b9261fad4ec63b2efbf9a80665f52def86282d26255e6331" +dependencies = [ + "flate2", + "http", + "log", + "native-tls", + "url", +] [[package]] name = "autocfg" @@ -64,6 +71,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +[[package]] +name = "autotools" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941527c41b0fc0dd48511a8154cd5fc7e29200a0ff8b7203c5d777dbc795cf" +dependencies = [ + "cc", +] + [[package]] name = "backtrace" version = "0.3.71" @@ -79,12 +95,41 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.5.0", + "cexpr", + "clang-sys", + "itertools", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.58", + "which", +] + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "bumpalo" version = "3.16.0" @@ -107,6 +152,15 @@ dependencies = [ "libc", ] +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -127,6 +181,17 @@ dependencies = [ "windows-targets 0.52.4", ] +[[package]] +name = "clang-sys" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" +dependencies = [ + "glob", + "libc", + "libloading", +] + [[package]] name = "cli-table" version = "0.4.7" @@ -150,12 +215,31 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" +[[package]] +name = "crc32fast" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +dependencies = [ + "cfg-if", +] + [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -249,6 +333,16 @@ dependencies = [ "quote", ] +[[package]] +name = "errno" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "eyre" version = "0.6.12" @@ -268,6 +362,55 @@ dependencies = [ "backtrace", ] +[[package]] +name = "fastrand" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" + +[[package]] +name = "filetime" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "windows-sys 0.52.0", +] + +[[package]] +name = "flate2" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -281,21 +424,26 @@ dependencies = [ name = "furiosa-device" version = "0.2.2-dev" dependencies = [ - "array_tool", + "attohttpc", + "autotools", + "bindgen", "cli-table", "dyn-clone", "enum-display-derive", "enum-utils", "eyre", + "flate2", "itertools", "lazy_static", "memoize", "nom", + "pkg-config", "rayon", "regex", "serde", "strum", "strum_macros", + "tar", "thiserror", "tokio", "tracing", @@ -426,7 +574,7 @@ version = "0.17.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044" dependencies = [ - "bitflags", + "bitflags 1.3.2", "libc", "libgit2-sys", "log", @@ -435,6 +583,12 @@ dependencies = [ "url", ] +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "hashbrown" version = "0.12.3" @@ -456,6 +610,26 @@ version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "home" +version = "0.5.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "iana-time-zone" version = "0.1.60" @@ -503,9 +677,9 @@ checksum = "bfa799dd5ed20a7e349f3b4639aa80d74549c81716d9ec4f994c9b5815598306" [[package]] name = "itertools" -version = "0.10.5" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" dependencies = [ "either", ] @@ -540,6 +714,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.153" @@ -560,6 +740,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "libloading" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +dependencies = [ + "cfg-if", + "windows-targets 0.52.4", +] + [[package]] name = "libssh2-sys" version = "0.3.0" @@ -586,6 +776,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" + [[package]] name = "lock_api" version = "0.4.11" @@ -637,9 +833,9 @@ dependencies = [ [[package]] name = "memoize" -version = "0.3.3" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c25d125e4063f313300d87c8f658e5b3d69257095df9a4221c12ba50b0421bff" +checksum = "5df4051db13d0816cf23196d3baa216385ae099339f5d0645a8d9ff2305e82b8" dependencies = [ "lazy_static", "lru", @@ -648,9 +844,9 @@ dependencies = [ [[package]] name = "memoize-inner" -version = "0.3.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7d5160e6ffcc59d4c571c38238ec5b7065bc91a5a24f511988dabcddda723" +checksum = "27bdece7e91f0d1e33df7b46ec187a93ea0d4e642113a1039ac8bfdd4a3273ac" dependencies = [ "lazy_static", "proc-macro2", @@ -684,6 +880,24 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "native-tls" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "7.1.3" @@ -738,6 +952,32 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "openssl" +version = "0.10.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" +dependencies = [ + "bitflags 2.5.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.58", +] + [[package]] name = "openssl-probe" version = "0.1.5" @@ -809,6 +1049,16 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "prettyplease" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d3928fb5db768cb86f891ff014f0144589297e3c6a1aba6ed7cecfdace270c7" +dependencies = [ + "proc-macro2", + "syn 2.0.58", +] + [[package]] name = "proc-macro2" version = "1.0.79" @@ -938,7 +1188,7 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -991,6 +1241,25 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65e04861e65f21776e67888bfbea442b3642beaa0138fdb1dd7a84a52dffdb89" +dependencies = [ + "bitflags 2.5.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + [[package]] name = "rustversion" version = "1.0.15" @@ -1003,12 +1272,44 @@ version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +[[package]] +name = "schannel" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" version = "1.0.197" @@ -1060,6 +1361,12 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + [[package]] name = "signal-hook-registry" version = "1.4.1" @@ -1096,21 +1403,21 @@ dependencies = [ [[package]] name = "strum" -version = "0.24.1" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063e6045c0e62079840579a7e47a355ae92f60eb74daaf156fb1e84ba164e63f" +checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" [[package]] name = "strum_macros" -version = "0.24.3" +version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" +checksum = "c6cf59daf282c0a494ba14fd21610a0325f9f90ec9d1231dea26bcb1d696c946" dependencies = [ "heck", "proc-macro2", "quote", "rustversion", - "syn 1.0.109", + "syn 2.0.58", ] [[package]] @@ -1135,12 +1442,35 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "tar" +version = "0.4.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b16afcea1f22891c49a00c751c7b63b2233284064f11a200fc624137c51e2ddb" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f" +[[package]] +name = "tempfile" +version = "3.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +dependencies = [ + "cfg-if", + "fastrand", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -1421,6 +1751,18 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix", +] + [[package]] name = "winapi" version = "0.3.9" @@ -1592,3 +1934,14 @@ name = "windows_x86_64_msvc" version = "0.52.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + +[[package]] +name = "xattr" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da84f1a25939b27f6820d92aed108f83ff920fdf11a7b19366c27c4cda81d4f" +dependencies = [ + "libc", + "linux-raw-sys", + "rustix", +] diff --git a/device-api/Cargo.toml b/device-api/Cargo.toml index 72136ab..d3ce66c 100644 --- a/device-api/Cargo.toml +++ b/device-api/Cargo.toml @@ -8,6 +8,7 @@ license = { workspace = true } homepage = { workspace = true } repository = { workspace = true } readme = { workspace = true } +links = "hwloc" [features] blocking = [] # Enable blocking APIs @@ -34,26 +35,37 @@ path = "bin/list_proc.rs" name = "list_clock_frequency" path = "bin/list_clock_frequency.rs" +[[bin]] +name = "show_topology" +path = "bin/show_topology.rs" + [dependencies] -array_tool = "1" cli-table = "0.4" dyn-clone = "1.0.17" enum-display-derive = "0.1" enum-utils = "0.1.2" -itertools = "0.10" +itertools = "0.12.1" lazy_static = "1.4" -memoize = { version = "0.3.1", features = ["full"] } +memoize = { version = "0.4.2", features = ["full"] } nom = "7.1" rayon = "1.5" regex = "1.5" serde = { version = "1.0.156", features = ["derive"] } -strum = "0.24" -strum_macros = "0.24" +strum = "0.26.2" +strum_macros = "0.26.2" thiserror = "1" tokio = { workspace = true } tracing = "0.1" tracing-subscriber = { version = "0.3.1", features = ["env-filter", "json"] } +[build-dependencies] +attohttpc = "0.28" +flate2 = "1.0" +tar = "0.4" +autotools = "0.2" +pkg-config = "0.3.8" +bindgen = "0.69.4" + [dev-dependencies] eyre = "0.6" diff --git a/device-api/bin/show_topology.rs b/device-api/bin/show_topology.rs new file mode 100644 index 0000000..7756abb --- /dev/null +++ b/device-api/bin/show_topology.rs @@ -0,0 +1,39 @@ +use cli_table::{print_stdout, Cell, Style, Table}; +use furiosa_device::{list_devices, topology::Topology, DeviceError}; + +#[tokio::main] +async fn main() -> Result<(), DeviceError> { + tracing_subscriber::fmt::init(); + + let devices = list_devices().await?; + if devices.is_empty() { + println!("No devices found."); + return Ok(()); + } + + let topology = Topology::new(devices.clone())?; + let mut rows = Vec::with_capacity(devices.len() + 1); + let mut header = Vec::with_capacity(devices.len() + 1); + header.push("Device".cell().bold(true)); + + for device in devices.iter() { + let name = device.to_string(); + header.push(name.cell().bold(true)); + } + rows.push(header); + + for device1 in devices.iter() { + let mut row = Vec::with_capacity(devices.len() + 1); + row.push(device1.to_string().cell()); + for device2 in devices.iter() { + let link_type = topology.get_link_type(device1, device2); + row.push(link_type.as_ref().cell()); + } + rows.push(row); + } + + let table = rows.table(); + print_stdout(table)?; + + Ok(()) +} diff --git a/device-api/build.rs b/device-api/build.rs new file mode 100644 index 0000000..51f253e --- /dev/null +++ b/device-api/build.rs @@ -0,0 +1,121 @@ +use std::env; +use std::path::{Path, PathBuf}; + +use bindgen::Builder; +use flate2::read::GzDecoder; +use tar::Archive; + +fn main() { + let version = "2.10.0"; + let out_path = env::var("OUT_DIR").expect("No output directory given"); + let source_path = fetch_hwloc(out_path, version); + let build_path = build_hwloc(&source_path); + gen_hwloc_binding(&build_path); + link_hwloc(&build_path); +} + +fn fetch_hwloc(parent_path: impl AsRef, version: &str) -> PathBuf { + let parent_path = parent_path.as_ref(); + let extracted_path = parent_path.join(format!("hwloc-{version}")); + + if extracted_path.exists() { + eprintln!("found hwloc v{version}"); + return extracted_path; + } + + let mut version_components = version.split('.'); + let major = version_components.next().expect("no major hwloc version"); + let minor = version_components.next().expect("no minor hwloc version"); + let url = format!( + "https://download.open-mpi.org/release/hwloc/v{major}.{minor}/hwloc-{version}.tar.gz" + ); + + eprintln!("Downloading hwloc v{version} from URL {url}..."); + let tar_gz = attohttpc::get(url) + .send() + .expect("failed to GET hwloc") + .bytes() + .expect("failed to parse HTTP response"); + + eprintln!("Extracting hwloc source..."); + let tar = GzDecoder::new(&tar_gz[..]); + let mut archive = Archive::new(tar); + archive + .unpack(parent_path) + .expect("failed to extract hwloc tar"); + + // Predict location where tarball was extracted + extracted_path +} + +fn build_hwloc(source_path: &Path) -> PathBuf { + let mut config = autotools::Config::new(source_path); + config.enable_static().disable_shared(); + // configure to use minimalistic XML backends + config.config_option("disable-libxml2", None); + + #[cfg(target_os = "macos")] + config.ldflag("-F/System/Library/Frameworks -framework CoreFoundation"); + + config.fast_build(true).reconf("-ivf").build() +} + +fn link_hwloc(install_path: &Path) { + let pkg_config_path = format!( + "{}:{}", + install_path.join("lib").join("pkgconfig").to_string_lossy(), + install_path + .join("lib64") + .join("pkgconfig") + .to_string_lossy(), + ); + env::set_var("PKG_CONFIG_PATH", pkg_config_path); + + let pkg_config = pkg_config::Config::new(); + let found = pkg_config.probe("hwloc").expect("couldn't find a hwloc"); + + for link_path in &found.link_paths { + println!( + "cargo:rustc-link-arg=-Wl,-rpath,{}", + link_path + .to_str() + .expect("Link path is not an UTF-8 string") + ); + + println!( + "cargo:rustc-link-search=native={}", + link_path + .to_str() + .expect("Link path is not an UTF-8 string") + ); + } + + println!("cargo:rustc-link-lib=static=hwloc"); +} + +fn gen_hwloc_binding(build_path: &Path) { + let include_path = build_path.join("include"); + let hwloc_include_path = include_path.join("hwloc"); + let hwloc_autogen_include_path = hwloc_include_path.join("autogen"); + + let bindings_file_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("hwloc_bindings.rs"); + + Builder::default() + .header(include_path.join("hwloc.h").to_str().unwrap()) + .clang_arg(format!("-I{}", include_path.to_str().unwrap())) + .clang_arg(format!("-I{}", hwloc_include_path.to_str().unwrap())) + .clang_arg(format!( + "-I{}", + hwloc_autogen_include_path.to_str().unwrap() + )) + .generate_inline_functions(true) + .generate() + .expect("Unable to generate bindings") + .write_to_file(bindings_file_path.clone()) + .expect("Couldn't write bindings!"); + + println!( + "hwloc bindings generated at {:?}", + bindings_file_path.display() + ); +} diff --git a/device-api/src/device.rs b/device-api/src/device.rs index ed3aa08..cab6391 100644 --- a/device-api/src/device.rs +++ b/device-api/src/device.rs @@ -11,7 +11,9 @@ use crate::arch::Arch; use crate::hwmon; use crate::perf_regs::PerformanceCounter; use crate::status::{get_device_status, DeviceStatus}; -use crate::sysfs::{npu_mgmt, pci}; +use crate::sysfs::npu_mgmt::{self}; +use crate::sysfs::pci; +use crate::topology::HardwareTopologyHint; use crate::{devfs, DeviceError, DeviceResult}; #[derive(Debug, Clone)] @@ -312,6 +314,12 @@ impl PartialOrd for Device { } } +impl HardwareTopologyHint for Device { + fn get_bpf(&self) -> String { + self.busname() + } +} + /// Non Uniform Memory Access (NUMA) node #[derive(Debug, Eq, PartialEq, Copy, Clone)] pub enum NumaNode { diff --git a/device-api/src/error.rs b/device-api/src/error.rs index 0e414b2..abb8d7b 100644 --- a/device-api/src/error.rs +++ b/device-api/src/error.rs @@ -37,6 +37,8 @@ pub enum DeviceError { UnexpectedValue { message: String }, #[error("Failed to parse given message {message}: {cause}")] ParseError { message: String, cause: String }, + #[error("Hwloc value: {message}")] + HwlocError { message: String }, } impl DeviceError { @@ -84,6 +86,12 @@ impl DeviceError { cause: cause.to_string(), } } + + pub(crate) fn hwloc_error(message: S) -> DeviceError { + DeviceError::HwlocError { + message: message.to_string(), + } + } } impl From for DeviceError { diff --git a/device-api/src/lib.rs b/device-api/src/lib.rs index 0bc916d..b07f10b 100644 --- a/device-api/src/lib.rs +++ b/device-api/src/lib.rs @@ -87,6 +87,7 @@ pub mod perf_regs; pub mod proc; mod status; mod sysfs; +pub mod topology; /// List all Furiosa NPU devices in the system. /// diff --git a/device-api/src/proc.rs b/device-api/src/proc.rs index 4f3bc5e..1f24762 100644 --- a/device-api/src/proc.rs +++ b/device-api/src/proc.rs @@ -99,6 +99,7 @@ fn read_cmdline(pid: u32) -> Option { #[cfg(test)] mod tests { + #[cfg(target_os = "linux")] use super::*; #[test] diff --git a/device-api/src/topology/hwloc.rs b/device-api/src/topology/hwloc.rs new file mode 100644 index 0000000..9841e90 --- /dev/null +++ b/device-api/src/topology/hwloc.rs @@ -0,0 +1,185 @@ +use std::ffi::CString; +use std::os::raw::c_int; +use std::ptr; + +use lazy_static::lazy_static; +use regex::Regex; + +use crate::topology::hwloc_binding::*; +use crate::{DeviceError, DeviceResult}; + +pub(crate) struct HwlocTopology { + topology: hwloc_topology_t, +} + +impl HwlocTopology { + pub(crate) fn new() -> Self { + Self { + topology: std::ptr::null_mut(), + } + } + + pub(crate) fn init_topology(&mut self) -> DeviceResult<()> { + if unsafe { hwloc_topology_init(&mut self.topology) } == 0 { + Ok(()) + } else { + Err(DeviceError::hwloc_error( + "couldn't initialize hwloc library", + )) + } + } + + pub(crate) fn set_io_types_filter(&mut self, filter: hwloc_type_filter_e) -> DeviceResult<()> { + if unsafe { hwloc_topology_set_io_types_filter(self.topology, filter) } == 0 { + Ok(()) + } else { + Err(DeviceError::hwloc_error("couldn't set filter")) + } + } + + pub(crate) fn load_topology(&mut self) -> DeviceResult<()> { + if unsafe { hwloc_topology_load(self.topology) } == 0 { + Ok(()) + } else { + Err(DeviceError::hwloc_error("couldn't load topology")) + } + } + + pub(crate) fn set_topology_from_xml(&mut self, xmlpath: &str) -> DeviceResult<()> { + let xml_path_cstr = CString::new(xmlpath).unwrap(); + if unsafe { hwloc_topology_set_xml(self.topology, xml_path_cstr.as_ptr()) } == 0 { + Ok(()) + } else { + Err(DeviceError::hwloc_error("couldn't set topology from xml")) + } + } + + pub(crate) fn get_common_ancestor_obj( + &self, + dev1bdf: &str, + dev2bdf: &str, + ) -> DeviceResult { + let dev1_obj = unsafe { hwloc_get_pcidev_by_busidstring(self.topology, dev1bdf) }; + if dev1_obj.is_null() { + return Err(DeviceError::hwloc_error(format!( + "couldn't find object with the bus id {dev1bdf}" + ))); + } + + let dev2_obj = unsafe { hwloc_get_pcidev_by_busidstring(self.topology, dev2bdf) }; + if dev2_obj.is_null() { + return Err(DeviceError::hwloc_error(format!( + "couldn't find object with the bus id {dev2bdf}" + ))); + } + + let ancestor = unsafe { hwloc_get_common_ancestor_obj(dev1_obj, dev2_obj) }; + if ancestor.is_null() { + return Err(DeviceError::hwloc_error(format!( + "couldn't find a common ancestor for objects {dev1bdf} and {dev2bdf}" + ))); + } + + Ok(ancestor) + } + + pub(crate) fn destroy_topology(&mut self) { + if !self.topology.is_null() { + unsafe { hwloc_topology_destroy(self.topology) }; + self.topology = std::ptr::null_mut(); + } + } +} + +unsafe fn hwloc_get_common_ancestor_obj( + mut obj1: hwloc_obj_t, + mut obj2: hwloc_obj_t, +) -> hwloc_obj_t { + while obj1 != obj2 { + while (*obj1).depth > (*obj2).depth { + obj1 = (*obj1).parent; + } + while (*obj2).depth > (*obj1).depth { + obj2 = (*obj2).parent; + } + if obj1 != obj2 && (*obj1).depth == (*obj2).depth { + obj1 = (*obj1).parent; + obj2 = (*obj2).parent; + } + } + obj1 +} + +unsafe fn hwloc_get_pcidev_by_busid( + topology: hwloc_topology_t, + domain: u16, + bus: u8, + dev: u8, + func: u8, +) -> hwloc_obj_t { + let mut obj = hwloc_get_next_pcidev(topology, ptr::null_mut()); + while obj != ptr::null_mut() { + if (*(*obj).attr).pcidev.domain == domain + && (*(*obj).attr).pcidev.bus == bus + && (*(*obj).attr).pcidev.dev == dev + && (*(*obj).attr).pcidev.func == func + { + return obj; + } + obj = hwloc_get_next_pcidev(topology, obj) + } + + ptr::null_mut() +} + +lazy_static! { + static ref BDF_NOTATION_PATTERN: Regex = Regex::new(r"^(?:(?P[0-9a-fA-F]+):)?(?P[0-9a-fA-F]+):(?P[0-9a-fA-F]+)\.(?P[0-9a-fA-F]+)").unwrap(); +} + +unsafe fn hwloc_get_pcidev_by_busidstring(topology: hwloc_topology_t, busid: &str) -> hwloc_obj_t { + return match BDF_NOTATION_PATTERN.captures(busid) { + Some(caps) => { + let domain = caps + .name("domain") + .map_or(0, |m| u16::from_str_radix(m.as_str(), 16).unwrap_or(0)); + let bus = u8::from_str_radix(caps.name("bus").unwrap().as_str(), 16).unwrap(); + let dev = u8::from_str_radix(caps.name("dev").unwrap().as_str(), 16).unwrap(); + let func = u8::from_str_radix(caps.name("func").unwrap().as_str(), 16).unwrap(); + hwloc_get_pcidev_by_busid(topology, domain, bus, dev, func) + } + None => ptr::null_mut(), + }; +} + +unsafe fn hwloc_get_next_obj_by_depth( + topology: hwloc_topology_t, + depth: c_int, + prev: hwloc_obj_t, +) -> hwloc_obj_t { + if prev.is_null() { + return hwloc_get_obj_by_depth(topology, depth, 0); + } + + if (*prev).depth != depth { + return ptr::null_mut(); + } + + (*prev).next_cousin +} + +unsafe fn hwloc_get_next_obj_by_type( + topology: hwloc_topology_t, + obj_type: hwloc_obj_type_t, + prev: hwloc_obj_t, +) -> hwloc_obj_t { + let depth = hwloc_get_type_depth(topology, obj_type); + return match depth { + hwloc_get_type_depth_e_HWLOC_TYPE_DEPTH_UNKNOWN => ptr::null_mut(), + hwloc_get_type_depth_e_HWLOC_TYPE_DEPTH_MULTIPLE => ptr::null_mut(), + d => hwloc_get_next_obj_by_depth(topology, d, prev), + }; +} + +unsafe fn hwloc_get_next_pcidev(topology: hwloc_topology_t, prev: hwloc_obj_t) -> hwloc_obj_t { + hwloc_get_next_obj_by_type(topology, hwloc_obj_type_t_HWLOC_OBJ_PCI_DEVICE, prev) +} diff --git a/device-api/src/topology/hwloc_binding.rs b/device-api/src/topology/hwloc_binding.rs new file mode 100644 index 0000000..0aa3021 --- /dev/null +++ b/device-api/src/topology/hwloc_binding.rs @@ -0,0 +1 @@ +include!(concat!(env!("OUT_DIR"), "/hwloc_bindings.rs")); diff --git a/device-api/src/topology/mod.rs b/device-api/src/topology/mod.rs new file mode 100644 index 0000000..cc082d1 --- /dev/null +++ b/device-api/src/topology/mod.rs @@ -0,0 +1,327 @@ +#![allow(warnings)] +use std::collections::BTreeMap; + +use itertools::iproduct; +use strum_macros::AsRefStr; + +use crate::topology::hwloc::HwlocTopology; +use crate::topology::hwloc_binding::*; +use crate::topology::LinkType::*; +use crate::{Device, DeviceResult}; + +mod hwloc; +mod hwloc_binding; + +#[derive(AsRefStr, Clone, Copy, Debug, PartialEq)] +pub enum LinkType { + // LinkTypeUnknown unknown + LinkTypeUnknown = 0, + // LinkTypeInterconnect two devices are connected across different cpus through interconnect. + LinkTypeInterconnect = 10, + // LinkTypeCPU two devices are connected under the same cpu, it may mean: + // devices are directly attached to the cpu pcie lane without PCIE switch. + // devices are attached to different PCIE switches under the same cpu. + LinkTypeCPU = 20, + // LinkTypeHostBridge two devices are connected under the same PCIE host bridge. + // Note that this does not guarantee devices are attached to the same PCIE switch. + // More switches could exist under the host bridge switch. + LinkTypeHostBridge = 30, + + // NOTE(@bg): Score 40 and 50 is reserved for LinkTypeMultiSwitch and LinkTypeSingleSwitch. + // NOTE(@bg): Score 60 is reserved for LinkTypeBoard + + // LinkTypeSoc two devices are on the same Soc chip. + LinkTypeSoc = 70, +} + +pub trait HardwareTopologyHint { + fn get_bpf(&self) -> String; +} + +pub struct Topology { + topology_matrix: BTreeMap<(String, String), LinkType>, +} + +impl Topology { + pub fn new(devices: Vec) -> DeviceResult { + let keys = devices.iter().map(|d| d.busname()).collect(); + let topology_provider = DefaultTopologyProvider::new()?; + let populated_matrix = populate_topology_matrix(topology_provider, keys)?; + Ok(Self { + topology_matrix: populated_matrix, + }) + } + + pub fn get_link_type(&self, device1: &T, device2: &T) -> LinkType { + self.get_link_type_with_bdf(device1.get_bpf(), device2.get_bpf()) + } + + fn get_link_type_with_bdf(&self, dev1_bdf: String, dev2_bdf: String) -> LinkType { + let key = if dev1_bdf > dev2_bdf { + (dev2_bdf, dev1_bdf) + } else { + (dev1_bdf, dev2_bdf) + }; + + match self.topology_matrix.get(&key) { + Some(link_type) => *link_type, + None => LinkTypeUnknown, + } + } +} + +fn populate_topology_matrix( + topology_provider: T, + devices: Vec, +) -> DeviceResult> { + let mut topology_matrix: BTreeMap<(String, String), LinkType> = BTreeMap::new(); + + for (dev1_bdf, dev2_bdf) in iproduct!(devices.clone(), devices.clone()) { + let link_type = if dev1_bdf == dev2_bdf { + LinkTypeSoc + } else { + topology_provider.get_common_ancestor_obj(&dev1_bdf, &dev2_bdf)? + }; + + let key = if dev1_bdf > dev2_bdf { + (dev2_bdf, dev1_bdf) + } else { + (dev1_bdf, dev2_bdf) + }; + + topology_matrix.entry(key).or_insert(link_type); + } + + Ok(topology_matrix) +} + +trait TopologyProvider { + fn get_common_ancestor_obj(&self, dev1_bdf: &str, dev2_bdf: &str) -> DeviceResult; +} + +struct DefaultTopologyProvider { + hwloc: HwlocTopology, +} + +impl TopologyProvider for DefaultTopologyProvider { + fn get_common_ancestor_obj(&self, dev1_bdf: &str, dev2_bdf: &str) -> DeviceResult { + let mut link = LinkTypeUnknown; + + if dev1_bdf == dev2_bdf { + link = LinkTypeSoc; + } else { + let ancestor = self.hwloc.get_common_ancestor_obj(dev1_bdf, dev2_bdf)?; + match unsafe { (*ancestor).type_ } { + hwloc_obj_type_t_HWLOC_OBJ_MACHINE => link = LinkTypeInterconnect, + hwloc_obj_type_t_HWLOC_OBJ_PACKAGE => link = LinkTypeCPU, + hwloc_obj_type_t_HWLOC_OBJ_BRIDGE => link = LinkTypeHostBridge, + _ => link = LinkTypeUnknown, + } + } + + Ok(link) + } +} + +impl DefaultTopologyProvider { + fn new() -> DeviceResult { + let mut hwloc = HwlocTopology::new(); + + // Initialize hwloc topology + hwloc.init_topology()?; + + // Set I/O types filter + hwloc.set_io_types_filter(hwloc_type_filter_e_HWLOC_TYPE_FILTER_KEEP_IMPORTANT)?; + + // Load the topology + hwloc.load_topology()?; + + Ok(Self { hwloc }) + } +} + +impl Drop for DefaultTopologyProvider { + fn drop(&mut self) { + self.hwloc.destroy_topology(); + } +} + +#[cfg(test)] +mod tests { + use std::env; + + use crate::topology::hwloc::HwlocTopology; + use crate::topology::hwloc_binding::*; + use crate::topology::LinkType::*; + use crate::topology::{ + populate_topology_matrix, DefaultTopologyProvider, LinkType, Topology, TopologyProvider, + }; + use crate::{Device, DeviceResult}; + + struct MockTopologyProvider { + provider: DefaultTopologyProvider, + } + + impl MockTopologyProvider { + fn new() -> DeviceResult { + let mut hwloc = HwlocTopology::new(); + hwloc.init_topology()?; + hwloc.set_io_types_filter(hwloc_type_filter_e_HWLOC_TYPE_FILTER_KEEP_IMPORTANT)?; + + let current_dir = env::current_dir().unwrap(); + let xml_path = current_dir.join("src/topology/test.xml"); + hwloc.set_topology_from_xml(xml_path.to_str().unwrap())?; + hwloc.load_topology()?; + + Ok(Self { + provider: DefaultTopologyProvider { hwloc }, + }) + } + } + + impl TopologyProvider for MockTopologyProvider { + fn get_common_ancestor_obj( + &self, + dev1_bdf: &str, + dev2_bdf: &str, + ) -> DeviceResult { + self.provider.get_common_ancestor_obj(dev1_bdf, dev2_bdf) + } + } + + #[test] + fn test_hwloc_init_and_destroy() { + let current_dir = env::current_dir().unwrap(); + let xml_path = current_dir.join("src/topology/test.xml"); + + let mut hwloc_topology = HwlocTopology::new(); + unsafe { + assert!(hwloc_topology.init_topology().is_ok()); + + assert!(hwloc_topology + .set_io_types_filter(hwloc_type_filter_e_HWLOC_TYPE_FILTER_KEEP_IMPORTANT) + .is_ok()); + + assert!(hwloc_topology + .set_topology_from_xml(xml_path.to_str().unwrap()) + .is_ok()); + + assert!(hwloc_topology.load_topology().is_ok()); + hwloc_topology.destroy_topology() + } + } + + // below hardware topology is used for testing + // Machine + // ├── Package (CPU) + // │ ├── Host Bridge (Root Complex) + // │ │ └── PCI Bridge + // │ │ └── PCI Bridge + // │ │ └── PCI Bridge + // │ │ └── PCI Bridge + // │ │ └── PCI Bridge + // │ │ └── PCI Bridge + // │ │ └── PCI Bridge + // │ │ ├── NPU0(0000:27:00.0) + // │ │ └── NPU1(0000:2a:00.0) + // │ └── Host Bridge (Root Complex) + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ ├── NPU2(0000:51:00.0) + // │ └── NPU3(0000:57:00.0) + // └── Package (CPU) + // ├── Host Bridge (Root Complex) + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ └── PCI Bridge + // │ ├── NPU4(0000:9e:00.0) + // │ └── NPU5(0000:a4:00.0) + // └── Host Bridge (Root Complex) + // └── PCI Bridge + // └── PCI Bridge + // └── PCI Bridge + // └── PCI Bridge + // └── PCI Bridge + // └── PCI Bridge + // └── PCI Bridge + // ├── NPU6(0000:c7:00.0) + // └── NPU7(0000:ca:00.0) + #[test] + fn test_topology_get_link_type() { + let keys = vec![ + "0000:27:00.0".to_string(), + "0000:2a:00.0".to_string(), + "0000:51:00.0".to_string(), + "0000:57:00.0".to_string(), + "0000:9e:00.0".to_string(), + "0000:a4:00.0".to_string(), + "0000:c7:00.0".to_string(), + "0000:ca:00.0".to_string(), + ]; + + let populated_matrix = + populate_topology_matrix(MockTopologyProvider::new().unwrap(), keys).unwrap(); + + let mut mock_topology = Topology { + topology_matrix: populated_matrix, + }; + + assert_eq!( + mock_topology.get_link_type_with_bdf(String::from(""), String::from("")), + LinkTypeUnknown + ); + assert_eq!( + mock_topology.get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("")), + LinkTypeUnknown + ); + assert_eq!( + mock_topology + .get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("0000:27:00.0")), + LinkTypeSoc + ); + assert_eq!( + mock_topology + .get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("0000:2a:00.0")), + LinkTypeHostBridge + ); + assert_eq!( + mock_topology + .get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("0000:51:00.0")), + LinkTypeCPU + ); + assert_eq!( + mock_topology + .get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("0000:57:00.0")), + LinkTypeCPU + ); + assert_eq!( + mock_topology + .get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("0000:9e:00.0")), + LinkTypeInterconnect + ); + assert_eq!( + mock_topology + .get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("0000:a4:00.0")), + LinkTypeInterconnect + ); + assert_eq!( + mock_topology + .get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("0000:c7:00.0")), + LinkTypeInterconnect + ); + assert_eq!( + mock_topology + .get_link_type_with_bdf(String::from("0000:27:00.0"), String::from("0000:ca:00.0")), + LinkTypeInterconnect + ); + } +} diff --git a/device-api/src/topology/test.xml b/device-api/src/topology/test.xml new file mode 100644 index 0000000..9f80d0d --- /dev/null +++ b/device-api/src/topology/test.xml @@ -0,0 +1,1252 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 0 1 + 10 20 20 10 + + \ No newline at end of file