Skip to content

Commit

Permalink
No public description
Browse files Browse the repository at this point in the history 
PiperOrigin-RevId: 707948343
  • Loading branch information
@copybara-github
MediaPipe Team authored and copybara-github committed Dec 19, 2024
1 parent 0f081a1 commit e00c842
Show file tree
Hide file tree
Showing 2 changed files with 238 additions and 4 deletions.
240 changes: 236 additions & 4 deletions mediapipe/framework/api2/node.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,227 @@
namespace mediapipe {
namespace api2 {

// Node (calculator / subgraph) interface.
//
// A subclass must declare its inputs, outputs, side inputs and side outputs
// (referred as "ports" below) using API2 types: Input, Output, SideInput,
// SideOutput.
//
// In addition, node can declare its timestamp offset and stream handler,
// using TimestampChange(...) and StreamHandler(...) respectively.
//
// To finish node interface declaration subclass must use
// MEDIAPIPE_NODE_INTERFACE macro.
//
// Example:
//
// class FooNode : public NodeIntf {
// public:
// static constexpr mediapipe::api2::Input<InputData> kInput{"IN"};
// static constexpr mediapipe::api2::Output<OutputData> kOutput{"OUT"};
//
// MEDIAPIPE_NODE_INTERFACE(FooNode, kInputData, kOutput);
// };
//
// Example overriding default timestamp offset (0 for NodeIntf/NodeImpl API)
// with the default of CalculatorBase - "arbitrary" (more details on this in
// NodeImpl documentation) and stream handler:
//
// class FooNode : public NodeIntf {
// public:
// static constexpr mediapipe::api2::Input<InputData> kInput{"IN"};
// static constexpr mediapipe::api2::Output<OutputData> kOutput{"OUT"};
//
// MEDIAPIPE_NODE_INTERFACE(FooNode, kInputData, kOutput,
// TimestampChange::Arbitrary(),
// StreamHandler("FixedSizeInputStreamHandler"));
// };
//
// NOTE: "IN" is a tag for FooNode input stream which can be used while
// authoring MediaPipe graph as CalculatorGraphConfig proto text, for example:
// input_stream: "in"
// node {
// calculator: "FooNode"
// input_stream: "IN:in"
// output_stream: "OUT:out"
// }
//
// or graph builder:
// Graph graph;
// Stream<InputData> in = graph.In(0).Cast<InputData>();
// auto& node = graph.AddNode("FooNode");
// in.ConnectTo(node.In("IN"));
// Stream<OutputData> out = node.Out("OUT").Cast<OutputData>();
//
// NOTE: it's recommended to provide meaningful tags for your node ports (e.g.
// helpful for debugging/logging purposes). In some cases you may still need to
// use empty string tags when migrating older calculators and keeping backward
// compatibility.
class NodeIntf {};

// Node (calculator) implementation.
//
// A subclass must specify node interface it implements as a first template
// parameter and itself as a second template parameter for the registration
// purposes.
//
// The subclass must implement Process() function and can implement Open() to
// do the initialization.
//
// For example:
// class FooNodeImpl : public NodeImpl<FooNode, FooNodeImpl> {
// public:
// absl::Status Process(CalculatorContext* cc) override {
// int input = kInput(cc).Get();
// ...
// }
// };
//
// Now FooNodeImpl is registered as "FooNode" (taken from `class FooNode...`) in
// MediaPipe registry automatically and can be used in CalculatorGraphConfig and
// Graph builder.
//
// Below is the explanation of how framework calls node functions:
//
// static absl::Status UpdateContract(CalculatorContract*)
// (Optional) Invoked on graph initialization, if defined, to update the
// contract.
//
// Then, for each run of the graph on a set of input side packets, the
// following sequence will occur.
//
// absl::Status Open(CalculatorContext*)
// (Optional) To initialize the calculator.
//
// NOTE : with the new API (NodeIntf/NodeImpl), the
// default Timestamp Offset of a calculator is 0. (Pay attention when
// migrating from older calculator APIs, because the default there is
// "arbitrary" Timestamp Offset.)
//
// With 0 Timestamp Offset, calculator is expected to send an output
// packet for every input packet at the input packet timestamp.
//
// If the calculator returns from Process without adding an output to some
// or all output streams:
// - The framework will send a timestamp bound update to downstream
// calculators that there won't be a packet for that particular
// timestamp on output streams in question.
// - Dependent downstream calculator(s) will execute on timestamp bound
// update if they have other input streams with ready packets at that
// particular timestamp. Input streams corresponding to output streams
// in question (with timestamp bound update) will have empty packets, so
// calculators need to use: IsEmpty before getting data.
//
// You can disable default 0 Timestamp Offset in the node interface or
// contract:
//
// MEDIAPIPE_NODE_INTERFACE(...,
// TimestampChange::Arbitrary());
// MEDIAPIPE_NODE_CONTRACT(...,
// TimestampChange::Arbitrary());
//
// NOTE: Clients can help optimize framework packet queueing by calling
// SetNextTimestampBound on outputs if applicable (e.g.
// kOutputPort(cc)->SetNextTimestampBound()).
//
// absl::Status Process(CalculatorContext*) (repeatedly)
//
// For Non-Source Nodes (nodes that have input streams):
//
// By default, invoked when every input stream either has a packet at
// timestamp T or the framework knows the packet is not expected at that
// timestamp. The latter occurs during timestamp bound update (Timestamp
// Offset is 0 (default), explicit call to SetNextTimestampBound() on
// calculator graph/upstream calculator or receiving a packet with a
// timestamp > T), and this results in corresponding input stream being
// empty during Process() call, so clients need to use: IsEmpty before
// getting data.
//
// This behavior may be adjusted, by utilizing different input stream
// handlers (please consult corresponding documentation):
// - DefaultInputStreamHandler (default)
// - FixedSizeInputStreamHandler
// - ImmediateInputStreamHandler
// - etc. in mediapipe/third_party/framework/stream_handler
//
// In the first place, consider setting it in the calculator if the
// calculator is required to have a custom stream handler always:
//
// MEDIAPIPE_NODE_INTERFACE(...,
// StreamHandler("FixedSizeInputStreamHandler"));
//
// Otherwise, you can set it in CalculatorGraphConfig:
// node {
// calculator: "CalculatorRunningAtOneFps"
// input_stream: "packets_streaming_in_at_ten_fps"
// input_stream_handler {
// input_stream_handler: "FixedSizeInputStreamHandler"
// }
// }
//
// or Graph builder:
// Graph graph;
// auto& node = graph.AddNode("CalculatorRunningAtOneFps");
// node.SetInputStreamHandler("FixedSizeInputStreamHandler")
//
// For Source Nodes (nodes that don't have input streams):
//
// Continues to have Process() called as long as it returns an
// absl::OkStatus(). Returning tool::StatusStop() indicates source node is
// done producing data.
//
// absl::Status Close(CalculatorContext*)
//
// After all calls to Process() finish or when all input streams close, the
// framework calls Close(). This function is always called if Open() was
// called and succeeded and even if the graph run terminated because of an
// error. No inputs are available via any input streams during Close(), but
// it still has access to input side packets and therefore may write
// outputs. After Close() returns, the calculator should be considered a
// dead node. The calculator object is destroyed as soon as the graph
// finishes running.
//
// NOTE: the entire calculator is constructed and destroyed for each graph run
// (set of input side packets, which could mean once per video, or once per
// image). Expensive operations and large objects should be input side packets
// or provided by graph services.
//
// Calculators must be thread-compatible.
// The framework does not call the non-const methods of a calculator from
// multiple threads at the same time. However, the thread that calls the methods
// of a calculator is not fixed. Therefore, calculators should not use
// ThreadLocal objects.
template <class Intf, class Impl = void>
class NodeImpl;

// Node (subgraph) implementation.
//
// A subclass must specify node interface it implements as a first template
// parameter and itself as a second template parameter for the registration
// purposes.
//
// The subclass must implement GetConfig() function to build the subgraph.
//
// For example:
// class FooSubgraphImpl : public SubgraphImpl<FooNode, FooSubgraphImpl> {
// public:
// absl::StatusOr<CalculatorGraphConfig> GetConfig(
// SubgraphContext* cc) override {
// Graph graph;
// ...
// return graph.GetConfig();
// }
// };
//
// Now FooSubgraphImpl is registered as "FooNode" (taken from
// `class FooNode...`) in MediaPipe registry automatically and can be used in
// CalculatorGraphConfig and Graph builder.
template <class Intf, class Impl>
class SubgraphImpl;

// SOFT DEPRECATION: use combination of NodeIntf/NodeImpl instead.
// TODO: deprecate with ABSL_DEPRECATED, migrate MP calculators to
// NodeIntf/NodeImpl.
class Node : public CalculatorBase {
public:
virtual ~Node();
Expand Down Expand Up @@ -72,6 +291,8 @@ MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(SubgraphRegistrator,

} // namespace internal

// FOR INTERNAL USE: use NodeIntf/NodeImpl instead.
//
// By passing the Impl parameter, registration is done automatically. No need
// to use MEDIAPIPE_NODE_IMPLEMENTATION.
// For backward compatibility, Impl can be omitted; use
Expand All @@ -80,9 +301,11 @@ MEDIAPIPE_STATIC_REGISTRATOR_TEMPLATE(SubgraphRegistrator,
template <class Impl = void>
class RegisteredNode;

// FOR INTERNAL USE: use NodeIntf/NodeImpl instead.
template <class Impl>
class RegisteredNode : public Node, private internal::NodeRegistrator<Impl> {};

// FOR INTERNAL USE: use NodeIntf/NodeImpl instead.
// No-op version for backwards compatibility.
template <>
class RegisteredNode<void> : public Node {};
Expand All @@ -94,7 +317,7 @@ struct FunctionNode : public RegisteredNode<Impl> {
}
};

template <class Intf, class Impl = void>
template <class Intf, class Impl>
class NodeImpl : public RegisteredNode<Impl>, public Intf {
protected:
// These methods allow accessing a node's ports by tag. This can be useful in
Expand Down Expand Up @@ -167,22 +390,31 @@ class SubgraphImpl : public Subgraph,
public Intf,
private internal::SubgraphRegistrator<Impl> {};

// DEPRECATED: use NodeIntf/NodeImpl and automatic registration it provides.
// Consult NodeIntf/NodeImpl for more details.
//
// This macro is used to register a calculator that does not use automatic
// registration. Deprecated.
// registration.
#define MEDIAPIPE_NODE_IMPLEMENTATION(Impl) \
MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \
mediapipe::CalculatorBaseRegistry, calculator_registration, \
Impl::kCalculatorName, \
std::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<Impl>>)

// This macro is used to register a non-split-contract calculator. Deprecated.
// DEPRECATED: use NodeIntf/NodeImpl and automatic registration it provides.
// Consult NodeIntf/NodeImpl for more details.
//
// This macro is used to register a non-split-contract calculator.
#define MEDIAPIPE_REGISTER_NODE(name) \
MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \
mediapipe::CalculatorBaseRegistry, calculator_registration, #name, \
std::make_unique<mediapipe::internal::CalculatorBaseFactoryFor<name>>)

// DEPRECATED: use NodeIntf/SubgraphImpl and automatic registration it provides.
// Consult NodeIntf/NodeImpl for more details.
//
// This macro is used to define a subgraph that does not use automatic
// registration. Deprecated.
// registration.
#define MEDIAPIPE_SUBGRAPH_IMPLEMENTATION(Impl) \
MEDIAPIPE_REGISTER_FACTORY_FUNCTION_QUALIFIED( \
mediapipe::SubgraphRegistry, subgraph_registration, \
Expand Down
2 changes: 2 additions & 0 deletions mediapipe/framework/calculator_base.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@

namespace mediapipe {

// SOFT DEPRECATION: use mediapipe::api2::NodeIntf/Impl instead.
//
// Experimental: CalculatorBase will eventually replace Calculator as the
// base class of leaf (non-subgraph) nodes in a CalculatorGraph.
//
Expand Down

0 comments on commit e00c842

Please sign in to comment.