Skip to content

Commit

Permalink
util/hist: Add high performance, fixed-size, exponential histogram
Browse files Browse the repository at this point in the history
  • Loading branch information
@ptaffet-jump
ptaffet-jump committed Nov 15, 2023
1 parent fdd3bb7 commit 6c88ba9
Show file tree
Hide file tree
Showing 3 changed files with 291 additions and 0 deletions.
3 changes: 3 additions & 0 deletions src/util/hist/Local.mk
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
$(call add-hdrs,fd_histf.h)
$(call make-unit-test,test_histf,test_histf,fd_util)
$(call run-unit-test,test_histf,)
165 changes: 165 additions & 0 deletions src/util/hist/fd_histf.h
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,165 @@
#ifndef HEADER_fd_src_util_hist_fd_histf_h
#define HEADER_fd_src_util_hist_fd_histf_h

/* Simple fast fixed-size exponential histograms. Histograms are
bucketed exponentially up to a maximum value, with an overflow bucket
for any other measurements. */

#include <math.h>
#include "../bits/fd_bits.h"
#include "../log/fd_log.h"
#if FD_HAS_AVX
#include "../simd/fd_avx.h"
#endif

#define FD_HISTF_BUCKET_CNT 16UL

#define FD_HISTF_ALIGN (32UL)
#define FD_HISTF_FOOTPRINT (FD_ULONG_ALIGN_UP( FD_HISTF_BUCKET_CNT*sizeof(ulong)+(FD_HISTF_BUCKET_CNT+1UL)*sizeof(long), FD_HISTF_ALIGN ))
/* Static assertion FOOTPRINT==sizeof in test */

struct __attribute__((aligned(FD_HISTF_ALIGN))) fd_histf_private {
ulong counts[ FD_HISTF_BUCKET_CNT ];
/* A value x belongs to bucket i if
left_edge[i] <= x - 2^63 < left_edge[i+1].
For AVX2, there's no unsiged comparison instruction. We follow
what wv_gt does and implement it by subtracting 2^63 from each
operand. Rather than perform the subtraction at each comparison,
we pre-subtract here. */
long left_edge[ FD_HISTF_BUCKET_CNT+1 ];
};

typedef struct fd_histf_private fd_histf_t;

FD_PROTOTYPES_BEGIN

FD_FN_CONST static inline ulong fd_histf_align ( void ) { return FD_HISTF_ALIGN; }
FD_FN_CONST static inline ulong fd_histf_footprint( void ) { return FD_HISTF_FOOTPRINT; }

/* fd_histf_new takes ownership of the memory region pointed to by mem
(which is assumed to be non-NULL with the appropriate alignment and
footprint) and formats it as a fd_hist. The histogram will be
initialized with buckets roughly exponentially spaced between
min_value and max_value. min_value must be > 0. Returns mem (which
will be formatted for use).
Every histogram has special buckets for underflow values (strictly
less than min_val) and overflow values (larger than or equal to the
max_value).
[ 0, min_value )
[ min_value, approx. min_value * z )
[ approx. min_value * z, approx. min_value * z^2 )
...
[ approx. min_value * z^13, max_value )
[ max_value, inf )
z is chosen so that max_value is approximately min_value * z^14 The
approximations come from the fact that all bucket edges are integers,
and no bucket is empty.
If max_value < min_value+14, then max_value will be increased to
min_value+14 so that no buckets are empty. Note that this histogram
contains strictly more information than what was requested, so an
end-user could postprocess and reduce the number of bins again
without losing any information.
For example, if min_value is 1 and max_value is 100, the buckets
will be
0: [ 0, 1)
1: [ 1, 2)
2: [ 2, 3)
3: [ 3, 4)
4: [ 4, 5)
5: [ 5, 7)
6: [ 7, 9)
7: [ 9, 12)
8: [ 12, 16)
9: [ 16, 22)
10: [ 22, 30)
11: [ 30, 41)
12: [ 41, 55)
13: [ 55, 74)
14: [ 74, 100)
15: [100, inf) */

static inline void *
fd_histf_new( void * mem,
ulong min_value,
ulong max_value ) {
if( FD_UNLIKELY( max_value<=min_value ) ) return NULL;

max_value = fd_ulong_max( max_value, min_value + FD_HISTF_BUCKET_CNT - 2UL );

fd_histf_t * hist = (fd_histf_t*)mem;
fd_memset( hist->counts, 0, FD_HISTF_BUCKET_CNT*sizeof(ulong) );
ulong left_edge[ FD_HISTF_BUCKET_CNT ]; /* without the -2^63 shift */
left_edge[ 0 ] = 0;
left_edge[ 1 ] = min_value;
for( ulong i=2UL; i<(FD_HISTF_BUCKET_CNT-1UL); i++ ) {
#if FD_HAS_DOUBLE
ulong le = (ulong)(0.5 + (double)left_edge[ i-1UL ] * pow ( (double)max_value / (double)left_edge[ i-1UL ], 1.0 /(double)(FD_HISTF_BUCKET_CNT - i) ) );
#else
ulong le = (ulong)(0.5f + (float )left_edge[ i-1UL ] * powf( (float )max_value / (float )left_edge[ i-1UL ], 1.0f/(float )(FD_HISTF_BUCKET_CNT - i) ) );
#endif
le = fd_ulong_max( le, left_edge[ i-1UL ] + 1UL ); /* Make sure bucket is not empty */
left_edge[ i ] = le;
}
left_edge[ FD_HISTF_BUCKET_CNT - 1UL ] = max_value;

for( ulong i=0UL; i<FD_HISTF_BUCKET_CNT; i++ ) hist->left_edge[ i ] = (long)(left_edge[ i ] - (1UL<<63));
hist->left_edge[ FD_HISTF_BUCKET_CNT ] = LONG_MAX;

return (void*)hist;
}

static inline fd_histf_t * fd_histf_join ( void * _hist ) { return (fd_histf_t *)_hist; }
static inline void * fd_histf_leave ( fd_histf_t * _hist ) { return (void *)_hist; }
static inline void * fd_histf_delete( void * _hist ) { return (void *)_hist; }

/* Return the number of buckets in the histogram, including the overflow
bucket. */
FD_FN_PURE static inline ulong fd_histf_bucket_cnt( fd_histf_t * hist ) { (void)hist; return FD_HISTF_BUCKET_CNT; }

/* Add a sample to the histogram. If the sample is larger than or equal
to the max_value it will be added to a special overflow bucket. */
static inline void
fd_histf_sample( fd_histf_t * hist,
ulong value ) {
long shifted_v = (long)(value - (1UL<<63));
#if FD_HAS_AVX
wl_t x = wl_bcast( shifted_v );
/* !(x-2^63 < left_edge[i]) & (x-2^63 < left_edge[i+1]) <=>
left_edge[i] <= x-2^63 < left_edge[i+1] */
wc_t select0 = wc_andnot( wl_lt( x, wl_ld ( hist->left_edge ) ),
wl_lt( x, wl_ldu( hist->left_edge+ 1UL ) ) );
wc_t select1 = wc_andnot( wl_lt( x, wl_ld ( hist->left_edge+ 4UL ) ),
wl_lt( x, wl_ldu( hist->left_edge+ 5UL ) ) );
wc_t select2 = wc_andnot( wl_lt( x, wl_ld ( hist->left_edge+ 8UL ) ),
wl_lt( x, wl_ldu( hist->left_edge+ 9UL ) ) );
wc_t select3 = wc_andnot( wl_lt( x, wl_ld ( hist->left_edge+12UL ) ),
wl_lt( x, wl_ldu( hist->left_edge+13UL ) ) );
/* In exactly one of these, we have a -1 (aka ULONG_MAX). We'll
subtract that from the counts, effectively adding 1. */
wv_st( hist->counts, wv_sub( wv_ld( hist->counts ), wc_to_wv_raw( select0 ) ) );
wv_st( hist->counts+ 4UL, wv_sub( wv_ld( hist->counts+ 4UL ), wc_to_wv_raw( select1 ) ) );
wv_st( hist->counts+ 8UL, wv_sub( wv_ld( hist->counts+ 8UL ), wc_to_wv_raw( select2 ) ) );
wv_st( hist->counts+12UL, wv_sub( wv_ld( hist->counts+12UL ), wc_to_wv_raw( select3 ) ) );
#else
for( ulong i=0UL; i<16UL; i++ ) hist->counts[ i ] += (hist->left_edge[ i ] <= shifted_v) & (shifted_v < hist->left_edge[ i+1UL ]);
#endif
}

/* Get the count of samples in a particular bucket of the historgram.
bucket in [0, 16) */
FD_FN_PURE static inline ulong
fd_histf_cnt( fd_histf_t * hist,
ulong bucket ) {
return hist->counts[ bucket ];
}

FD_PROTOTYPES_END

#endif /* HEADER_fd_src_util_hist_fd_histf_h */
123 changes: 123 additions & 0 deletions src/util/hist/test_histf.c
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
#include "../fd_util.h"
#include "fd_histf.h"
#include "../rng/fd_rng.h"
#include <math.h>
#include <stdlib.h>

FD_STATIC_ASSERT( FD_HISTF_ALIGN ==alignof(fd_histf_t), unit_test );
FD_STATIC_ASSERT( FD_HISTF_FOOTPRINT==sizeof (fd_histf_t), unit_test );

static inline void
assert_range( fd_histf_t * hist,
ulong idx,
uint left_edge,
uint right_edge ) { /* exclusive */
ulong expected = fd_histf_cnt( hist, idx );
fd_histf_sample( hist, left_edge-1U ); /* Might underflow, but okay */
FD_TEST( fd_histf_cnt( hist, idx )==expected );

for( uint i=left_edge; i<right_edge; i++ ) {
fd_histf_sample( hist, i );
FD_TEST( fd_histf_cnt( hist, idx )==++expected );
}
fd_histf_sample( hist, right_edge );
FD_TEST( fd_histf_cnt( hist, idx )==expected );
}

int
main( int argc,
char ** argv ) {
fd_boot( &argc, &argv );

FD_LOG_NOTICE(( "Testing align / footprint" ));

FD_TEST( fd_histf_align ()==FD_HISTF_ALIGN );
FD_TEST( fd_histf_footprint()==FD_HISTF_FOOTPRINT );

FD_LOG_NOTICE(( "Testing new" ));

fd_histf_t * _hist = aligned_alloc( FD_HISTF_ALIGN, FD_HISTF_FOOTPRINT ); FD_TEST( !!_hist );
void * shhist = fd_histf_new( _hist, 4U, 20U ); FD_TEST( !!shhist );

FD_LOG_NOTICE(( "Testing join" ));

fd_histf_t * hist = fd_histf_join( shhist ); FD_TEST( !!hist );

FD_LOG_NOTICE(( "Testing sample" ));

for( ulong i=0; i<16UL; i++ ) FD_TEST( fd_histf_cnt( hist, i )==0UL );

fd_histf_sample( hist, 0U );

FD_TEST( fd_histf_cnt( hist, 0 )==1UL );
for( ulong i=1UL; i<16UL; i++ ) FD_TEST( fd_histf_cnt( hist, i )==0UL );

/* All < 4 so go in underflow bucket */
fd_histf_sample( hist, 1U );
fd_histf_sample( hist, 2U );
fd_histf_sample( hist, 3U );

FD_TEST( fd_histf_cnt( hist, 0UL )==4UL );
for( ulong i=1UL; i<16UL; i++ ) FD_TEST( fd_histf_cnt( hist, i )==0UL );

fd_histf_sample( hist, 20U ); FD_TEST( fd_histf_cnt( hist, 15UL )==1UL );
fd_histf_sample( hist, 21U ); FD_TEST( fd_histf_cnt( hist, 15UL )==2UL );
fd_histf_sample( hist, 30U ); FD_TEST( fd_histf_cnt( hist, 15UL )==3UL );
fd_histf_sample( hist, 99U ); FD_TEST( fd_histf_cnt( hist, 15UL )==4UL );

hist = fd_histf_join( fd_histf_new( fd_histf_delete( fd_histf_leave( hist ) ), 1U, 100U ) );

assert_range( hist, 0UL, 0U, 1U );
assert_range( hist, 1UL, 1U, 2U );
assert_range( hist, 2UL, 2U, 3U );
assert_range( hist, 3UL, 3U, 4U );
assert_range( hist, 4UL, 4U, 5U );
assert_range( hist, 5UL, 5U, 7U );
assert_range( hist, 6UL, 7U, 9U );
assert_range( hist, 7UL, 9U, 12U );
assert_range( hist, 8UL, 12U, 16U );
assert_range( hist, 9UL, 16U, 22U );
assert_range( hist, 10UL, 22U, 30U );
assert_range( hist, 11UL, 30U, 41U );
assert_range( hist, 12UL, 41U, 55U );
assert_range( hist, 13UL, 55U, 74U );
assert_range( hist, 14UL, 74U, 100U );
/* We've already tested the overflow bucket above */

FD_LOG_NOTICE(( "Testing bucket_cnt" ));

FD_TEST( fd_histf_bucket_cnt( hist )==FD_HISTF_BUCKET_CNT );

FD_LOG_NOTICE(( "Testing performance" ));
fd_rng_t _rng[1]; fd_rng_t * rng = fd_rng_join( fd_rng_new( _rng, 0U, 0UL ) );
long overhead = -fd_log_wallclock();
for( ulong i=0UL; i<1000000000UL; i++ ) {
uint v = fd_rng_uint_roll( rng, 100U );
FD_COMPILER_FORGET( v );
}
overhead += fd_log_wallclock();

long time = -fd_log_wallclock();
for( ulong i=0UL; i<1000000000UL; i++ ) {
uint v = fd_rng_uint_roll( rng, 100U );
fd_histf_sample( hist, v );
}
time += fd_log_wallclock();

FD_LOG_NOTICE(( "average time per sample %f ns (excluding rng overhead)",
(double)(time - overhead)/1000000000.0 ));

FD_LOG_NOTICE(( "Testing leave" ));

FD_TEST( fd_histf_leave( hist )==shhist );

FD_LOG_NOTICE(( "Testing delete" ));

FD_TEST( fd_histf_delete( shhist )==_hist );
free( _hist );

FD_LOG_NOTICE(( "pass" ));
fd_halt();
return 0;
}

0 comments on commit 6c88ba9

Please sign in to comment.