From 209234a1828af58b65082dbb8f0aa22230cc58b3 Mon Sep 17 00:00:00 2001 From: Mingwei Zhang Date: Sun, 11 Feb 2024 10:25:09 -0800 Subject: [PATCH] prepare 0.10.0 release --- CHANGELOG.md | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- README.md | 81 ++++++++++++++++++++++++++----------------- src/lib.rs | 79 +++++++++++++++++++++++++----------------- 4 files changed, 195 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9182e83..51cea0b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,103 @@ All notable changes to this project will be documented in this file. +## v0.10.0 - 2024-02-12 + +Version 0.10.0 is a major release with a lot of changes and improvements. + +### Highlights + +#### MRT Encoding + +`bgpkit-parser` now supports encoding MRT messages. The following MRT message types are supported: +- TableDumpV1 +- TableDumpV2 +- BGP4MP +It also supports encoding BMP messages into MRT files. + +Example of writing BgpElems into a file RIB dump file: +```rust +let mut encoder = bgpkit_parser::encoder::MrtRibEncoder::new(); +for elem in parser { + encoder.process_elem(&elem); +} +let mut writer = oneio::get_writer("filtered.rib.gz").unwrap(); +writer.write_all(encoder.export_bytes().as_ref()).unwrap(); +drop(writer); +``` + +Another example of writing `BgpElem` to BGP updates bytes: +```rust +let mut encoder = bgpkit_parser::encoder::MrtUpdatesEncoder::new(); +let mut elem = BgpElem::default(); +elem.peer_ip = IpAddr::V4("10.0.0.1".parse().unwrap()); +elem.peer_asn = Asn::from(65000); +elem.prefix.prefix = "10.250.0.0/24".parse().unwrap(); +encoder.process_elem(&elem); +elem.prefix.prefix = "10.251.0.0/24".parse().unwrap(); +encoder.process_elem(&elem); +let bytes = encoder.export_bytes(); + +let mut cursor = Cursor::new(bytes.clone()); +while cursor.has_remaining() { +let parsed = parse_mrt_record(&mut cursor).unwrap(); + dbg!(&parsed); +} +``` + +See `encoder` module for more details. + +#### Better developer experiences + +- added several utility functions to `BgpElem` + - `.is_announcement()`: check if the BGP element is an announcement + - `.get_as_path_opt()`: get the AS path if it exists and no AS set or confederated segments + - `.get_origin_asn_opt()`: get the origin ASN if it exists +- full `serde` serialization support +- add `BgpElem` to PSV (pipe-separated values) conversion +- improved time-related filters parsing + - `ts_start` `start_ts` `ts_end` `end_ts` are all supported +- many quality of life improvements by [@jmeggitt](https://github.com/jmeggitt) + - https://github.com/bgpkit/bgpkit-parser/pull/122 + - https://github.com/bgpkit/bgpkit-parser/pull/123 + +#### Improved testing coverage + +- `bgpkit-parser` code test coverage is now at 92%. + - codecov.io coverage report is available at https://app.codecov.io/gh/bgpkit/bgpkit-parser + +#### TLS backend choice + +- by default, `bgpkit-parser` now uses `rustls` as the default TLS backend + - `openssl` is still supported, but it is no longer the default + - `openssl` support can be enabled by using the `native-tls` feature flag and set default features to `false` + +### Added RFCs support + +- [`RFC 4724: Graceful Restart Mechanism for BGP`][rfc4724] +- [`RFC 8671 Support for Adj-RIB-Out in the BGP Monitoring Protocol (BMP)`][rfc8671] +- [`RFC 9069: Support for Local RIB in the BGP Monitoring Protocol (BMP)`][rfc9069] +- the supported RFCs list is documented at https://github.com/bgpkit/bgpkit-parser?tab=readme-ov-file#rfcs-support + +[rfc4724]: https://www.rfc-editor.org/rfc/rfc4724 +[rfc8671]: https://www.rfc-editor.org/rfc/rfc8671 +[rfc9069]: https://www.rfc-editor.org/rfc/rfc9069 + +### Fixes + +- fixed a bug where when multiple `AsSequences` are present, only the first one is parsed + - issue: https://github.com/bgpkit/bgpkit-parser/issues/140 +- fixed a bug where the parser panics when messages are truncated + - https://github.com/bgpkit/bgpkit-parser/issues/149 + +### Other changes + +- Move pybgpkit to its own repository at https://github.com/bgpkit/pybgpkit +- CLI build feature changed from `build-binary` to `cli` +- add more ways to install compiled `bgpkit-parser` CLI + - homebrew on macOS: `brew install bgpkit/tap/bgpkit-parser` + - other platforms: `cargo binstall bgpkit-parser` + ## v0.10.0-beta.3 - 2024-02-08 ### Highlights diff --git a/Cargo.toml b/Cargo.toml index 40a1c36..72182e0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "bgpkit-parser" -version = "0.10.0-beta.3" +version = "0.10.0" authors = ["Mingwei Zhang "] edition = "2021" license = "MIT" diff --git a/README.md b/README.md index 9b9758e..eb30c15 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # BGPKIT Parser -*This readme is generated from the library's doc comments using [cargo-readme](https://github.com/livioribeiro/cargo-readme). Please refer to the Rust docs website for the full documentation: [latest stable](https://docs.rs/bgpkit-parser/latest/bgpkit_parser/); [bleeding-edge](https://docs.rs/bgpkit-parser/0.10.0-beta.3/bgpkit_parser/).* +*This readme is generated from the library's doc comments using [cargo-readme](https://github.com/livioribeiro/cargo-readme). Please refer to the Rust docs website for the full documentation: [latest stable](https://docs.rs/bgpkit-parser/latest/bgpkit_parser/); [bleeding-edge](https://docs.rs/bgpkit-parser/0.10.0/bgpkit_parser/).* [![Build](https://github.com/bgpkit/bgpkit-parser/actions/workflows/build.yml/badge.svg)](https://github.com/bgpkit/bgpkit-parser/actions/workflows/build.yml) [![Crates.io](https://img.shields.io/crates/v/bgpkit-parser)](https://crates.io/crates/bgpkit-parser) @@ -55,7 +55,7 @@ total: 255849 [BGPKIT Broker][broker-repo] library provides search API for all RouteViews and RIPE RIS MRT data files. Using the broker's Rust API ([`bgpkit-broker`][broker-crates-io]), we can easily compile a list of MRT files that we are interested in for any time period and any data type (`update` or `rib`). This allows users to gather information without needing to -know about locations of specific data files. +know about the locations of specific data files. [broker-repo]: https://github.com/bgpkit/bgpkit-broker [broker-crates-io]: https://crates.io/crates/bgpkit-broker @@ -100,7 +100,7 @@ for item in broker.into_iter().take(2) { ### Filtering BGP Messages -BGPKIT Parser also has built-in [Filter] mechanism. When creating a new [`BgpkitParser`] instance, +BGPKIT Parser also has a built-in [Filter] mechanism. When creating a new [`BgpkitParser`] instance, once can also call `add_filter` function to customize the parser to only show matching messages when iterating through [BgpElem]s. @@ -109,7 +109,7 @@ For all types of filters, check out the [Filter] enum documentation. ```rust use bgpkit_parser::BgpkitParser; -/// This example shows how to parse a MRT file and filter by prefix. +/// This example shows how to parse an MRT file and filter by prefix. env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); log::info!("downloading updates file"); @@ -243,47 +243,64 @@ drop(mrt_writer); `bgpkit-parser` is bundled with a utility commandline tool `bgpkit-parser-cli`. +### Installation + +#### Install compiled binaries + +You can install the compiled `bgpkit-parser` CLI binaries with the following methods: +- **Homebrew** (macOS): `brew install bgpkit/tap/bgpkit-parser` +- [**Cargo binstall**](binstall): `cargo binstall bgpkit-parser` + +[binstall]: https://github.com/cargo-bins/cargo-binstall + +#### From source + You can install the tool by running ```bash cargo install bgpkit-parser --features cli ``` or checkout this repository and run ```bash -cargo install --path ./bgpkit-parser --features cli +cargo install --path . --features cli ``` +### Usage + +Run `bgpkit-parser --help` to see the full list of options. + ``` -➜ cli git:(cli) ✗ bgpkit-parser-cli 0.1.0 - -Mingwei Zhang - -bgpkit-parser-cli is a simple cli tool that allow parsing of individual MRT files - -USAGE: - bgpkit-parser-cli [FLAGS] [OPTIONS] - -FLAGS: - -e, --elems-count Count BGP elems - -h, --help Prints help information - --json Output as JSON objects - --pretty Pretty-print JSON output - -r, --records-count Count MRT records - -V, --version Prints version information - -OPTIONS: - -a, --as-path Filter by AS path regex string - -m, --elem-type Filter by elem type: announce (a) or withdraw (w) - -T, --end-ts Filter by end unix timestamp inclusive - -o, --origin-asn Filter by origin AS Number - -J, --peer-asn Filter by peer IP ASN - -j, --peer-ip Filter by peer IP address - -p, --prefix Filter by network prefix - -t, --start-ts Filter by start unix timestamp inclusive +MRT/BGP/BMP data processing library + +Usage: bgpkit-parser [OPTIONS] + +Arguments: + File path to a MRT file, local or remote + +Options: + -c, --cache-dir Set the cache directory for caching remote files. Default behavior does not enable caching + --json Output as JSON objects + --psv Output as full PSV entries with header + --pretty Pretty-print JSON output + -e, --elems-count Count BGP elems + -r, --records-count Count MRT records + -o, --origin-asn Filter by origin AS Number + -p, --prefix Filter by network prefix + -s, --include-super Include super-prefix when filtering + -S, --include-sub Include sub-prefix when filtering + -j, --peer-ip Filter by peer IP address + -J, --peer-asn Filter by peer ASN + -m, --elem-type Filter by elem type: announce (a) or withdraw (w) + -t, --start-ts Filter by start unix timestamp inclusive + -T, --end-ts Filter by end unix timestamp inclusive + -a, --as-path Filter by AS path regex string + -h, --help Print help + -V, --version Print version + ``` ## Data Representation -There are two key data structure to understand for the parsing results: [MrtRecord] and [BgpElem]. +There are two key data structures to understand for the parsing results: [MrtRecord] and [BgpElem]. ### `MrtRecord`: unmodified MRT information representation diff --git a/src/lib.rs b/src/lib.rs index a34cd17..4e04f1a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -45,7 +45,7 @@ total: 255849 [BGPKIT Broker][broker-repo] library provides search API for all RouteViews and RIPE RIS MRT data files. Using the broker's Rust API ([`bgpkit-broker`][broker-crates-io]), we can easily compile a list of MRT files that we are interested in for any time period and any data type (`update` or `rib`). This allows users to gather information without needing to -know about locations of specific data files. +know about the locations of specific data files. [broker-repo]: https://github.com/bgpkit/bgpkit-broker [broker-crates-io]: https://crates.io/crates/bgpkit-broker @@ -90,7 +90,7 @@ for item in broker.into_iter().take(2) { ## Filtering BGP Messages -BGPKIT Parser also has built-in [Filter] mechanism. When creating a new [`BgpkitParser`] instance, +BGPKIT Parser also has a built-in [Filter] mechanism. When creating a new [`BgpkitParser`] instance, once can also call `add_filter` function to customize the parser to only show matching messages when iterating through [BgpElem]s. @@ -99,7 +99,7 @@ For all types of filters, check out the [Filter] enum documentation. ```no_run use bgpkit_parser::BgpkitParser; -/// This example shows how to parse a MRT file and filter by prefix. +/// This example shows how to parse an MRT file and filter by prefix. env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); log::info!("downloading updates file"); @@ -235,47 +235,64 @@ drop(mrt_writer); `bgpkit-parser` is bundled with a utility commandline tool `bgpkit-parser-cli`. +## Installation + +### Install compiled binaries + +You can install the compiled `bgpkit-parser` CLI binaries with the following methods: +- **Homebrew** (macOS): `brew install bgpkit/tap/bgpkit-parser` +- [**Cargo binstall**](binstall): `cargo binstall bgpkit-parser` + +[binstall]: https://github.com/cargo-bins/cargo-binstall + +### From source + You can install the tool by running ```bash cargo install bgpkit-parser --features cli ``` or checkout this repository and run ```bash -cargo install --path ./bgpkit-parser --features cli +cargo install --path . --features cli ``` +## Usage + +Run `bgpkit-parser --help` to see the full list of options. + ```text -➜ cli git:(cli) ✗ bgpkit-parser-cli 0.1.0 - -Mingwei Zhang - -bgpkit-parser-cli is a simple cli tool that allow parsing of individual MRT files - -USAGE: - bgpkit-parser-cli [FLAGS] [OPTIONS] - -FLAGS: - -e, --elems-count Count BGP elems - -h, --help Prints help information - --json Output as JSON objects - --pretty Pretty-print JSON output - -r, --records-count Count MRT records - -V, --version Prints version information - -OPTIONS: - -a, --as-path Filter by AS path regex string - -m, --elem-type Filter by elem type: announce (a) or withdraw (w) - -T, --end-ts Filter by end unix timestamp inclusive - -o, --origin-asn Filter by origin AS Number - -J, --peer-asn Filter by peer IP ASN - -j, --peer-ip Filter by peer IP address - -p, --prefix Filter by network prefix - -t, --start-ts Filter by start unix timestamp inclusive +MRT/BGP/BMP data processing library + +Usage: bgpkit-parser [OPTIONS] + +Arguments: + File path to a MRT file, local or remote + +Options: + -c, --cache-dir Set the cache directory for caching remote files. Default behavior does not enable caching + --json Output as JSON objects + --psv Output as full PSV entries with header + --pretty Pretty-print JSON output + -e, --elems-count Count BGP elems + -r, --records-count Count MRT records + -o, --origin-asn Filter by origin AS Number + -p, --prefix Filter by network prefix + -s, --include-super Include super-prefix when filtering + -S, --include-sub Include sub-prefix when filtering + -j, --peer-ip Filter by peer IP address + -J, --peer-asn Filter by peer ASN + -m, --elem-type Filter by elem type: announce (a) or withdraw (w) + -t, --start-ts Filter by start unix timestamp inclusive + -T, --end-ts Filter by end unix timestamp inclusive + -a, --as-path Filter by AS path regex string + -h, --help Print help + -V, --version Print version + ``` # Data Representation -There are two key data structure to understand for the parsing results: [MrtRecord] and [BgpElem]. +There are two key data structures to understand for the parsing results: [MrtRecord] and [BgpElem]. ## `MrtRecord`: unmodified MRT information representation