From e00c842ec44f33eaa1c177c18ca160b17dc14924 Mon Sep 17 00:00:00 2001 From: MediaPipe Team Date: Thu, 19 Dec 2024 10:05:43 -0800 Subject: [PATCH] No public description PiperOrigin-RevId: 707948343 --- mediapipe/framework/api2/node.h | 240 +++++++++++++++++++++++++- mediapipe/framework/calculator_base.h | 2 + 2 files changed, 238 insertions(+), 4 deletions(-) diff --git a/mediapipe/framework/api2/node.h b/mediapipe/framework/api2/node.h index ca6a049cf4..1edbf2dd26 100644 --- a/mediapipe/framework/api2/node.h +++ b/mediapipe/framework/api2/node.h @@ -15,8 +15,227 @@ namespace mediapipe { namespace api2 { +// Node (calculator / subgraph) interface. +// +// A subclass must declare its inputs, outputs, side inputs and side outputs +// (referred as "ports" below) using API2 types: Input, Output, SideInput, +// SideOutput. +// +// In addition, node can declare its timestamp offset and stream handler, +// using TimestampChange(...) and StreamHandler(...) respectively. +// +// To finish node interface declaration subclass must use +// MEDIAPIPE_NODE_INTERFACE macro. +// +// Example: +// +// class FooNode : public NodeIntf { +// public: +// static constexpr mediapipe::api2::Input kInput{"IN"}; +// static constexpr mediapipe::api2::Output kOutput{"OUT"}; +// +// MEDIAPIPE_NODE_INTERFACE(FooNode, kInputData, kOutput); +// }; +// +// Example overriding default timestamp offset (0 for NodeIntf/NodeImpl API) +// with the default of CalculatorBase - "arbitrary" (more details on this in +// NodeImpl documentation) and stream handler: +// +// class FooNode : public NodeIntf { +// public: +// static constexpr mediapipe::api2::Input kInput{"IN"}; +// static constexpr mediapipe::api2::Output kOutput{"OUT"}; +// +// MEDIAPIPE_NODE_INTERFACE(FooNode, kInputData, kOutput, +// TimestampChange::Arbitrary(), +// StreamHandler("FixedSizeInputStreamHandler")); +// }; +// +// NOTE: "IN" is a tag for FooNode input stream which can be used while +// authoring MediaPipe graph as CalculatorGraphConfig proto text, for example: +// input_stream: "in" +// node { +// calculator: "FooNode" +// input_stream: "IN:in" +// output_stream: "OUT:out" +// } +// +// or graph builder: +// Graph graph; +// Stream in = graph.In(0).Cast(); +// auto& node = graph.AddNode("FooNode"); +// in.ConnectTo(node.In("IN")); +// Stream out = node.Out("OUT").Cast(); +// +// NOTE: it's recommended to provide meaningful tags for your node ports (e.g. +// helpful for debugging/logging purposes). In some cases you may still need to +// use empty string tags when migrating older calculators and keeping backward +// compatibility. class NodeIntf {}; +// Node (calculator) implementation. +// +// A subclass must specify node interface it implements as a first template +// parameter and itself as a second template parameter for the registration +// purposes. +// +// The subclass must implement Process() function and can implement Open() to +// do the initialization. +// +// For example: +// class FooNodeImpl : public NodeImpl { +// public: +// absl::Status Process(CalculatorContext* cc) override { +// int input = kInput(cc).Get(); +// ... +// } +// }; +// +// Now FooNodeImpl is registered as "FooNode" (taken from `class FooNode...`) in +// MediaPipe registry automatically and can be used in CalculatorGraphConfig and +// Graph builder. +// +// Below is the explanation of how framework calls node functions: +// +// static absl::Status UpdateContract(CalculatorContract*) +// (Optional) Invoked on graph initialization, if defined, to update the +// contract. +// +// Then, for each run of the graph on a set of input side packets, the +// following sequence will occur. +// +// absl::Status Open(CalculatorContext*) +// (Optional) To initialize the calculator. +// +// NOTE : with the new API (NodeIntf/NodeImpl), the +// default Timestamp Offset of a calculator is 0. (Pay attention when +// migrating from older calculator APIs, because the default there is +// "arbitrary" Timestamp Offset.) +// +// With 0 Timestamp Offset, calculator is expected to send an output +// packet for every input packet at the input packet timestamp. +// +// If the calculator returns from Process without adding an output to some +// or all output streams: +// - The framework will send a timestamp bound update to downstream +// calculators that there won't be a packet for that particular +// timestamp on output streams in question. +// - Dependent downstream calculator(s) will execute on timestamp bound +// update if they have other input streams with ready packets at that +// particular timestamp. Input streams corresponding to output streams +// in question (with timestamp bound update) will have empty packets, so +// calculators need to use: IsEmpty before getting data. +// +// You can disable default 0 Timestamp Offset in the node interface or +// contract: +// +// MEDIAPIPE_NODE_INTERFACE(..., +// TimestampChange::Arbitrary()); +// MEDIAPIPE_NODE_CONTRACT(..., +// TimestampChange::Arbitrary()); +// +// NOTE: Clients can help optimize framework packet queueing by calling +// SetNextTimestampBound on outputs if applicable (e.g. +// kOutputPort(cc)->SetNextTimestampBound()). +// +// absl::Status Process(CalculatorContext*) (repeatedly) +// +// For Non-Source Nodes (nodes that have input streams): +// +// By default, invoked when every input stream either has a packet at +// timestamp T or the framework knows the packet is not expected at that +// timestamp. The latter occurs during timestamp bound update (Timestamp +// Offset is 0 (default), explicit call to SetNextTimestampBound() on +// calculator graph/upstream calculator or receiving a packet with a +// timestamp > T), and this results in corresponding input stream being +// empty during Process() call, so clients need to use: IsEmpty before +// getting data. +// +// This behavior may be adjusted, by utilizing different input stream +// handlers (please consult corresponding documentation): +// - DefaultInputStreamHandler (default) +// - FixedSizeInputStreamHandler +// - ImmediateInputStreamHandler +// - etc. in mediapipe/third_party/framework/stream_handler +// +// In the first place, consider setting it in the calculator if the +// calculator is required to have a custom stream handler always: +// +// MEDIAPIPE_NODE_INTERFACE(..., +// StreamHandler("FixedSizeInputStreamHandler")); +// +// Otherwise, you can set it in CalculatorGraphConfig: +// node { +// calculator: "CalculatorRunningAtOneFps" +// input_stream: "packets_streaming_in_at_ten_fps" +// input_stream_handler { +// input_stream_handler: "FixedSizeInputStreamHandler" +// } +// } +// +// or Graph builder: +// Graph graph; +// auto& node = graph.AddNode("CalculatorRunningAtOneFps"); +// node.SetInputStreamHandler("FixedSizeInputStreamHandler") +// +// For Source Nodes (nodes that don't have input streams): +// +// Continues to have Process() called as long as it returns an +// absl::OkStatus(). Returning tool::StatusStop() indicates source node is +// done producing data. +// +// absl::Status Close(CalculatorContext*) +// +// After all calls to Process() finish or when all input streams close, the +// framework calls Close(). This function is always called if Open() was +// called and succeeded and even if the graph run terminated because of an +// error. No inputs are available via any input streams during Close(), but +// it still has access to input side packets and therefore may write +// outputs. After Close() returns, the calculator should be considered a +// dead node. The calculator object is destroyed as soon as the graph +// finishes running. +// +// NOTE: the entire calculator is constructed and destroyed for each graph run +// (set of input side packets, which could mean once per video, or once per +// image). Expensive operations and large objects should be input side packets +// or provided by graph services. +// +// Calculators must be thread-compatible. +// The framework does not call the non-const methods of a calculator from +// multiple threads at the same time. However, the thread that calls the methods +// of a calculator is not fixed. Therefore, calculators should not use +// ThreadLocal objects. +template +class NodeImpl; + +// Node (subgraph) implementation. +// +// A subclass must specify node interface it implements as a first template +// parameter and itself as a second template parameter for the registration +// purposes. +// +// The subclass must implement GetConfig() function to build the subgraph. +// +// For example: +// class FooSubgraphImpl : public SubgraphImpl { +// public: +// absl::StatusOr GetConfig( +// SubgraphContext* cc) override { +// Graph graph; +// ... +// return graph.GetConfig(); +// } +// }; +// +// Now FooSubgraphImpl is registered as "FooNode" (taken from +// `class FooNode...`) in MediaPipe registry automatically and can be used in +// CalculatorGraphConfig and Graph builder. +template +class SubgraphImpl; + +// SOFT DEPRECATION: use combination of NodeIntf/NodeImpl instead. +// TODO: deprecate with ABSL_DEPRECATED, migrate MP calculators to +// NodeIntf/NodeImpl. class Node : public CalculatorBase { public: virtual ~Node(); @@ -72,6 +291,8 @@ MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(SubgraphRegistrator, } // namespace internal +// FOR INTERNAL USE: use NodeIntf/NodeImpl instead. +// // By passing the Impl parameter, registration is done automatically. No need // to use MEDIAPIPE_NODE_IMPLEMENTATION. // For backward compatibility, Impl can be omitted; use @@ -80,9 +301,11 @@ MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(SubgraphRegistrator, template class RegisteredNode; +// FOR INTERNAL USE: use NodeIntf/NodeImpl instead. template class RegisteredNode : public Node, private internal::NodeRegistrator {}; +// FOR INTERNAL USE: use NodeIntf/NodeImpl instead. // No-op version for backwards compatibility. template <> class RegisteredNode : public Node {}; @@ -94,7 +317,7 @@ struct FunctionNode : public RegisteredNode { } }; -template +template class NodeImpl : public RegisteredNode, public Intf { protected: // These methods allow accessing a node's ports by tag. This can be useful in @@ -167,22 +390,31 @@ class SubgraphImpl : public Subgraph, public Intf, private internal::SubgraphRegistrator {}; +// DEPRECATED: use NodeIntf/NodeImpl and automatic registration it provides. +// Consult NodeIntf/NodeImpl for more details. +// // This macro is used to register a calculator that does not use automatic -// registration. Deprecated. +// registration. #define MEDIAPIPE_NODE_IMPLEMENTATION(Impl) \ MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \ mediapipe::CalculatorBaseRegistry, calculator_registration, \ Impl::kCalculatorName, \ std::make_unique>) -// This macro is used to register a non-split-contract calculator. Deprecated. +// DEPRECATED: use NodeIntf/NodeImpl and automatic registration it provides. +// Consult NodeIntf/NodeImpl for more details. +// +// This macro is used to register a non-split-contract calculator. #define MEDIAPIPE_REGISTER_NODE(name) \ MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \ mediapipe::CalculatorBaseRegistry, calculator_registration, #name, \ std::make_unique>) +// DEPRECATED: use NodeIntf/SubgraphImpl and automatic registration it provides. +// Consult NodeIntf/NodeImpl for more details. +// // This macro is used to define a subgraph that does not use automatic -// registration. Deprecated. +// registration. #define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl) \ MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \ mediapipe::SubgraphRegistry, subgraph_registration, \ diff --git a/mediapipe/framework/calculator_base.h b/mediapipe/framework/calculator_base.h index 1f4c82160e..b7fc912bc1 100644 --- a/mediapipe/framework/calculator_base.h +++ b/mediapipe/framework/calculator_base.h @@ -31,6 +31,8 @@ namespace mediapipe { +// SOFT DEPRECATION: use mediapipe::api2::NodeIntf/Impl instead. +// // Experimental: CalculatorBase will eventually replace Calculator as the // base class of leaf (non-subgraph) nodes in a CalculatorGraph. //