Concurrent FW 2.5 release (#500)

Includes:
* Introduction of IndexedPcapReader, seek metehod to PcapReader
* Moving port guessing logic from Python into C++
* `SensorHTTP` class made public
* experimental mtp_init_client to init client for multicast support
* Support of future DF sensor line metadata via introduction of fps
  and changes to metadata parsing
* Numerous viz changes including introducing palettes, ImageMode and
  CloudMode
* bugfix on application of autoleveling for viz
* FindPcap updates
* FW 2.5 test metadata assets
* Change default metadata output to nonlegacy

CHANGELOG.rst

@@ -2,12 +2,40 @@
 Changelog
 =========
 
-[unreleased]
-============
+[20230403]
+==========
+
+* Default metadata output across all functionality has been switched to the non-legacy format
 
 ouster_client
 -------------
-- Added a new method ``mtp_init_client`` to init the client with multicast support (experimental).
+* Added a new method ``mtp_init_client`` to init the client with multicast support (experimental).
+* the class ``SensorHttp``  which provides easy access to REST APIs of the sensor has been made public
+  under the ``ouster::sensor::util`` namespace.
+* breaking change: get_metadata defaults to outputting non-legacy metadata
+* add debug five_word profile which will be removed later
+* breaking change: remove deprecations on LidarScan
+
+ouster_viz
+----------
+* update viz camera with other objects in draw
+
+ouster_pcap
+-----------
+* add seek method to PcapReader
+* add port guessing logic
+
+python
+------
+* introduce utility to convert nonlegacy metadata to legacy
+* use resolve_metadata to find unspecified metadata for simple-viz
+* remove port guessing logic in favor of using new C++ ouster_pcap port guessing functionality
+* add soft-id-check to skip the init_id/sn check for lidar_packets with metadata
+
+Numerous changes to SimpleViz and LidarScanViz including:
+* expose visible in viz to Python 
+* introduce ImageMode and CloudMode
+* bugfix: remove spurious sqrt application to autoleveled images
 
 
 [20230114]
@@ -90,10 +118,6 @@ ouster_client
 * allow vcpkg configuration via environment variables
 * fix a bug in sensor_config struct equality comparison operator
 
-ouster_pcap
------------
-* fix incorrect encapsulation protocol being reported in ``packet_info``
-
 ouster_viz
 ----------
 * clean up GL context logic to avoid errors on window/intel UHD graphics

CMakeLists.txt

@@ -7,10 +7,10 @@ include(DefaultBuildType)
 include(VcpkgEnv)
 
 # ==== Project Name ====
-project(ouster_example VERSION 20230114)
+project(ouster_example VERSION 20230403)
 
 # generate version header
-set(OusterSDK_VERSION_STRING 0.7.1)
+set(OusterSDK_VERSION_STRING 0.8.1)
 include(VersionGen)
 
 # ==== Options ====
@@ -89,7 +89,6 @@ set(CPACK_DEBIAN_PACKAGE_NAME ouster-sdk)
 set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT)
 set(CPACK_DEBIAN_PACKAGE_DEPENDS
   "libjsoncpp-dev, libeigen3-dev, libtins-dev, libglfw3-dev, libglew-dev, libspdlog-dev")
-
 include(CPack)
 
 # ==== Install ====

cmake/FindPcap.cmake

@@ -0,0 +1,24 @@
+find_package(PkgConfig QUIET)
+if(PKG_CONFIG_FOUND)
+  pkg_check_modules(PC_PCAP QUIET libpcap)
+  if(PC_PCAP_FOUND)
+    set(PCAP_VERSION_STRING ${PC_PCAP_VERSION})
+  endif()
+endif()
+
+find_path(PCAP_INCLUDE_DIR
+  NAMES pcap.h
+  HINTS ${PC_PCAP_INCLUDE_DIRS})
+
+find_library(PCAP_LIBRARY
+  NAMES pcap pcap_static wpcap
+  HINTS ${PC_PCAP_LIBRARY_DIRS})
+
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Pcap
+  REQUIRED_VARS PCAP_LIBRARY PCAP_INCLUDE_DIR
+  VERSION_VAR PCAP_VERSION_STRING)
+
+mark_as_advanced(
+  PCAP_INCLUDE_DIR
+  PCAP_LIBRARY)

conan/test_package/conanfile.py

@@ -2,6 +2,7 @@
 
 from conans import ConanFile, CMake, tools
 
+
 class OusterSDKTestConan(ConanFile):
     settings = "os", "compiler", "build_type", "arch"
     generators = "cmake_paths", "cmake_find_package"

conanfile.py

@@ -4,6 +4,7 @@
 
 from pprint import pformat
 
+
 class OusterSDKConan(ConanFile):
     name = "ouster_sdk"
     license = "BSD 3-Clause License"
@@ -48,7 +49,7 @@ class OusterSDKConan(ConanFile):
     # https://docs.conan.io/en/1.51/howtos/capture_version.html#how-to-capture-package-version-from-text-or-build-files
     def set_version(self):
         content = tools.load(os.path.join(self.recipe_folder, "CMakeLists.txt"))
-        version = re.search("set\(OusterSDK_VERSION_STRING (.*)\)", content).group(1)
+        version = re.search(r"set\(OusterSDK_VERSION_STRING (.*)\)", content).group(1)
         self.version = version.strip()
 
     def config_options(self):
@@ -65,7 +66,7 @@ def requirements(self):
         self.requires("eigen/3.4.0")
         self.requires("jsoncpp/1.9.5")
         self.requires("spdlog/1.10.0")
-        self.requires("libcurl/7.82.0")
+        self.requires("libcurl/7.84.0")
 
         if self.options.build_pcap:
             self.requires("libtins/4.3")

docs/Doxyfile

@@ -845,7 +845,6 @@ WARN_LOGFILE           =
 INPUT                  = ../ouster_client \
                          ../ouster_pcap \
                          ../ouster_viz \
-                         ../ouster_ros
 
 # This tag can be used to specify the character encoding of the source files
 # that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses

docs/conf.py

@@ -22,7 +22,6 @@
 from string import Template
 import shutil
 import os
-import sys
 import re
 
 project = 'Ouster Sensor SDK'
@@ -40,13 +39,15 @@
     raise RuntimeError("Could not guess OUSTER_SDK_PATH")
 print(OUSTER_SDK_PATH)
 
+
 # https://packaging.python.org/en/latest/guides/single-sourcing-package-version/
 def parse_version():
     with open(os.path.join(OUSTER_SDK_PATH, 'CMakeLists.txt')) as listfile:
         content = listfile.read()
-        groups = re.search("set\(OusterSDK_VERSION_STRING ([^-\)]+)(-(.*))?\)", content)
+        groups = re.search(r"set\(OusterSDK_VERSION_STRING ([^-\)]+)(-(.*))?\)", content)
         return groups.group(1) + (groups.group(3) or "")
 
+
 # The full version, including alpha/beta/rc tags
 version = release = parse_version()
 
@@ -203,13 +204,14 @@ def do_doxygen_generate_xml(app):
         app.config["breathe_projects"].update(
             {name: os.path.join(doxygen_output_dir, path)})
 
+
 def do_doxygen_temp_cleanup(app, exception):
     if "temp_doxy_file_dir" in app.config:
         shutil.rmtree(app.config["temp_doxy_file_dir"])
 
+
 def setup(app):
 
     # Add a hook for generating doxygen xml and cleaning up
     app.connect("builder-inited", do_doxygen_generate_xml)
     app.connect("build-finished", do_doxygen_temp_cleanup)
-

docs/cpp/ouster_client/lidar_scan.rst

@@ -29,7 +29,7 @@ XYZLut
 .. doxygenstruct:: ouster::XYZLut
     :members:
 
-.. doxygenfunction:: ouster::make_xyz_lut(size_t w, size_t h, double range_unit, double lidar_origin_to_beam_origin_mm, const mat4d& transform, const std::vector<double>& azimuth_angles_deg, const std::vector<double>& altitude_angles_deg)
+.. doxygenfunction:: ouster::make_xyz_lut(size_t w, size_t h, double range_unit, const mat4d& beam_to_lidar_transform, const mat4d& transform, const std::vector<double>& azimuth_angles_deg, const std::vector<double>& altitude_angles_deg)
 
 .. doxygenfunction:: ouster::make_xyz_lut(const sensor::sensor_info& sensor)
 

docs/cpp/ouster_pcap/os_pcap.rst

@@ -33,10 +33,12 @@ Functions
 
 .. doxygenfunction:: ouster::sensor_utils::read_packet
 
-.. doxygenfunction:: ouster::sensor_utils::record_initialize
+.. doxygenfunction:: ouster::sensor_utils::record_initialize( const std::string& file, const std::string& src_ip, const std::string& dst_ip, int frag_size, bool use_sll_encapsulation = false)
+
+.. doxygenfunction:: ouster::sensor_utils::record_initialize(const std::string& file, int frag_size, bool use_sll_encapsulation = false);
 
 .. doxygenfunction:: ouster::sensor_utils::record_uninitialize
 
-.. doxygenfunction:: ouster::sensor_utils::record_packet(record_handle& handle, const packet_info& info, const uint8_t* buf, size_t buffer_size)
+.. doxygenfunction:: ouster::sensor_utils::record_packet(record_handle& handle, const std::string& src_ip, const std::string& dst_ip, int src_port, int dst_port, const uint8_t* buf, size_t buffer_size, uint64_t microsecond_timestamp)
 
 

docs/index.rst

@@ -38,6 +38,12 @@
    C++ API Reference <cpp/api>
    Changelog <reference/changelog>
 
+.. toctree::
+   :caption: Migration Guides
+   :hidden:
+
+    Migrating from 20220927/0.5.1 to 20230114/0.7.1 <migration/migration-20220927-20230114>
+    Migrating from 20230114/0.7.1 to 20230403/0.8.1 <migration/migration-20230114-20230403>
 ..
    FAQ <faq>
 

docs/migration/migration-20220927-20230114.rst

@@ -0,0 +1,101 @@
+===============================================
+Migration from 20220927/0.5.1 to 20230114/0.7.1
+===============================================
+
+The 20230114 release, which corresponds to Python SDK 0.7.1, brings a few breaking changes
+introduced to support the newest FW of the sensor, FW 3.0. The changes and how to mitigate them are
+summarized here.
+
+Signal Multiplier
+-----------------
+
+FW 3.0 allows setting the signal multiplier to non-int values 0.25 and 0.5. As such, we have changed
+the type of the ``sensor_config`` struct member ``signal_multiplier`` to double. 
+
+Old saved config jsons will continue to load properly.
+
+XYZLut and ``sensor_info``
+--------------------------
+
+The ``sensor_info`` struct now contains the ``beam_to_lidar_transform`` specifying the relationship
+between the beam coordinate frame and lidar coordinate frame. The double
+``lidar_origin_to_beam_origin_mm`` which corresponds to the (0, 3) element of the
+``beam_to_lidar_transform`` has not been removed.
+
+The ``make_xyz_lut`` function now takes a ``mat4d`` transformation specifying the relation between
+the beam and lidar coordinate frames, as opposed to the  previous double which assumed an Identity
+rotation. 
+
+Users using pre-FW3.0 sensors with the latest SDK ``make_xyz_lut`` overload which accepts a
+``sensor_info`` need not change anything, as the ``sensor_info`` struct automatically derives and
+self-populates with the appropriate ``beam_to_lidar_transform``.
+
+Default Parameters in init_client
+---------------------------------
+
+The shortform C++ ``init_client`` and Python ``client.Sensor()`` no longer have default parameters.
+Since they do not configure the sensor, it didn't make sense to default to any value for any
+parameters. Users must provide the ports and hostname explicitly now.
+
+Timeout Improvements
+--------------------
+
+The Python Scans interface timeout parameter has been changed from None to 1 second by default to
+avoid confusing hanging behavior when timing out is appropriate. The default timeout has also been
+changed to 2 seconds across the board.
+
+Notes for the future
+--------------------
+
+For customers who know they will continue to upgrade their version of the SDK, we also wish to
+highlight some upcoming breaking changes.
+
+Default Metadata Format
++++++++++++++++++++++++
+First, the next release will default writing the non-legacy metadata format as opposed to the
+current format, also known as the legacy metadata format. The SDK will continue to read the legacy
+format, i.e., it will continue to read old recorded data), and it will also be able to produce the
+legacy format if the parameter ``legacy=true`` is specified to the ``get_metadata`` function.
+
+Deprecations
+++++++++++++
+We are going to start removing a number of deprecations in the ``ouster_client`` code with the next
+release. We will document any such removals and how to migrate from them in this or the following
+migration guide (whichever is relevant).
+
+ - ``LidarScan::N_FIELDS``: The number of fields in a profile has varied since the introduction of
+   eUDP profiles. To find the number of fields in your scan, we suggest using the iterator to loop
+   over and count the number of fields.
+
+
+ - ``LidarScan::Field``: use ``sensor::ChanField`` instead.
+
+ - ``LidarScan::BlockHeader``: ``BlockHeaders``, structs consisting of the ``timestamp``,
+   ``encoder`` ticks, and ``status`` which corresponded to a single column of measurements
+   (identifiable by``measurement_id``), have been deprecated in favor of ``Headers``, which are
+   Eigen Arrays of length equivalent to the # of columns in a scan, with the ith element being the
+   value from the ith measurement. As such, where once you might write:
+
+        .. code::
+
+            auto n_invalid = std::count_if( scans[i].headers.begin(), scans[i].headers.end(), [](const
+            LidarScan::BlockHeader& h) { return h.status != 0xffffffff; });
+
+  you should now use the ``status()`` function:
+
+        .. code::
+
+            auto status = scan.status(); auto n_invalid = std::count_if(status.data(), status.data() +
+            status.size(), [](const uint32_t s) { return !(s & 0x01); });
+
+  Timestamps are now available through ``timestamp()``, also returning a ``Header``; and the information
+  contained in ``encoder`` is available through ``measurement_id()`` (see the next line item for the
+   conversion).
+
+- ``encoder``: Encoder counts were part of the LEGACY UDP profile, representing the azimuth angle as
+  a raw encoder count starting from 0 to a max value of 90111. It thus incremented 44 ticks per
+  azimuth angle in 2048 mode, 88 ticks in 1024 mode and 176 in 512 mode. To recover encoder
+  infcount, you can multiply the ``measurement_id`` by the number dictated by your lidar mode. We
+  would suggest, however, migrating to use simply ``measurement_id`` and multiplying by ``360
+  degrees/ N`` where ``N`` is the number of columns in your mode (512, 1024, 2048) when you need the
+  azimuth, thus untying any sense of azimuth from the internal mechanicals of the Ouster sensor.