Skip to content

Commit

Permalink
security: add remote signing tile for shreds
Browse files Browse the repository at this point in the history
  • Loading branch information
mmcgee-jump committed Jan 22, 2024
1 parent a57d59d commit df9ddc2
Show file tree
Hide file tree
Showing 21 changed files with 695 additions and 147 deletions.
2 changes: 1 addition & 1 deletion config/everything.mk
Original file line number Diff line number Diff line change
Expand Up @@ -399,7 +399,7 @@ LLVM_PROFILE_FILE="$(OBJDIR)/cov/raw/integration_tests.profraw" \
contrib/test/run_integration_tests.sh

seccomp-policies:
$(FIND) . -name '*.seccomppolicy' -exec $(PYTHON) contrib/test/generate_filters.py {} \;
$(FIND) . -name '*.seccomppolicy' -exec $(PYTHON) contrib/codegen/generate_filters.py {} \;

##############################
# LLVM Coverage
Expand Down
3 changes: 2 additions & 1 deletion src/app/fdctl/Local.mk
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ ifdef FD_HAS_DOUBLE

.PHONY: fdctl cargo rust solana

$(call add-objs,main1 config caps utility topology keys ready mem spy help run/run run/tiles/tiles run/run1 run/run_solana run/tiles/tiles run/tiles/fd_net run/tiles/fd_metric run/tiles/fd_netmux run/tiles/fd_dedup run/tiles/fd_pack run/tiles/fd_quic run/tiles/fd_verify run/tiles/fd_poh run/tiles/fd_bank run/tiles/fd_shred run/tiles/fd_store monitor/monitor monitor/helper configure/configure configure/large_pages configure/sysctl configure/shmem configure/xdp configure/xdp_leftover configure/ethtool configure/workspace_leftover configure/workspace,fd_fdctl)
$(call add-objs,main1 config caps utility topology keys ready mem spy help run/run run/tiles/tiles run/run1 run/run_solana run/tiles/tiles run/tiles/fd_net run/tiles/fd_metric run/tiles/fd_netmux run/tiles/fd_dedup run/tiles/fd_pack run/tiles/fd_quic run/tiles/fd_verify run/tiles/fd_poh run/tiles/fd_bank run/tiles/fd_shred run/tiles/fd_store run/tiles/fd_sign monitor/monitor monitor/helper configure/configure configure/large_pages configure/sysctl configure/shmem configure/xdp configure/xdp_leftover configure/ethtool configure/workspace_leftover configure/workspace,fd_fdctl)
$(call make-bin-rust,fdctl,main,fd_fdctl fd_disco fd_flamenco fd_ip fd_reedsol fd_ballet fd_tango fd_util fd_quic solana_validator)
$(OBJDIR)/obj/app/fdctl/configure/xdp.o: src/tango/xdp/fd_xdp_redirect_prog.o
$(OBJDIR)/obj/app/fdctl/config.o: src/app/fdctl/config/default.toml
Expand All @@ -19,6 +19,7 @@ $(OBJDIR)/obj/app/fdctl/run/tiles/fd_quic.o: src/app/fdctl/run/tiles/generated/q
$(OBJDIR)/obj/app/fdctl/run/tiles/fd_shred.o: src/app/fdctl/run/tiles/generated/shred_seccomp.h
$(OBJDIR)/obj/app/fdctl/run/tiles/fd_verify.o: src/app/fdctl/run/tiles/generated/verify_seccomp.h
$(OBJDIR)/obj/app/fdctl/run/tiles/fd_metric.o: src/app/fdctl/run/tiles/generated/metric_seccomp.h
$(OBJDIR)/obj/app/fdctl/run/tiles/fd_sign.o: src/app/fdctl/run/tiles/generated/sign_seccomp.h

# Phony target to always rerun cargo build ... it will detect if anything
# changed on the library side.
Expand Down
88 changes: 56 additions & 32 deletions src/app/fdctl/config.c

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion src/app/fdctl/config/default.toml
Original file line number Diff line number Diff line change
Expand Up @@ -496,7 +496,7 @@ dynamic_port_range = "8900-9000"
#
# It is suggested to use all available CPU cores for Firedancer, so
# that the Solana network can run as fast as possible.
affinity = "0-22"
affinity = "0-23"

# How many net tiles to run. Each networking tile will service
# exactly one queue from a network device being listened to. If
Expand Down
8 changes: 6 additions & 2 deletions src/app/fdctl/run/run1.c
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,11 @@ tile_main( void * _args ) {
const fd_frag_meta_t * in_mcache[ FD_TOPO_MAX_LINKS ];
ulong * in_fseq[ FD_TOPO_MAX_TILE_IN_LINKS ];

ulong polled_in_cnt = 0UL;
for( ulong i=0; i<tile->in_cnt; i++ ) {
if( FD_UNLIKELY( !tile->in_link_poll[ i ] ) ) continue;

polled_in_cnt += 1;
in_mcache[ i ] = args->config->topo.links[ tile->in_link_id[ i ] ].mcache;
FD_TEST( in_mcache[ i ] );
in_fseq[ i ] = tile->in_link_fseq[ i ];
Expand All @@ -136,7 +140,7 @@ tile_main( void * _args ) {
for( ulong i=0; i<args->config->topo.tile_cnt; i++ ) {
fd_topo_tile_t * consumer_tile = &args->config->topo.tiles[ i ];
for( ulong j=0; j<consumer_tile->in_cnt; j++ ) {
if( FD_UNLIKELY( consumer_tile->in_link_id[ j ] == tile->out_link_id_primary && consumer_tile->in_link_reliable[ j ] ) ) {
if( FD_UNLIKELY( consumer_tile->in_link_id[ j ]==tile->out_link_id_primary && consumer_tile->in_link_reliable[ j ] ) ) {
out_fseq[ out_cnt_reliable ] = consumer_tile->in_link_fseq[ j ];
FD_TEST( out_fseq[ out_cnt_reliable ] );
out_cnt_reliable++;
Expand Down Expand Up @@ -167,7 +171,7 @@ tile_main( void * _args ) {
fd_rng_t rng[1];
fd_mux_tile( tile->cnc,
config->mux_flags,
tile->in_cnt,
polled_in_cnt,
in_mcache,
in_fseq,
tile->out_link_id_primary == ULONG_MAX ? NULL : args->config->topo.links[ tile->out_link_id_primary ].mcache,
Expand Down
46 changes: 36 additions & 10 deletions src/app/fdctl/run/tiles/fd_shred.c
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@
#define POH_IN_IDX 1
#define STAKE_IN_IDX 2
#define CONTACT_IN_IDX 3
#define SIGN_IN_IDX 4

#define NET_OUT_IDX 0
#define SIGN_OUT_IDX 1

#define MAX_SLOTS_PER_EPOCH 432000UL

Expand Down Expand Up @@ -119,11 +123,12 @@ typedef struct {
fd_fec_resolver_t * resolver;
fd_pubkey_t identity_key[1]; /* Just the public key */

fd_remote_signer_t remote_signer_ctx[1];

uint src_ip_addr;
uchar src_mac_addr[ 6 ];
ushort shred_listen_port;
fd_ip_t * ip;
uchar const * shred_signing_key;

/* shred34 and fec_sets are very related: fec_sets[i] has pointers
to the shreds in shred34[4*i + k] for k=0,1,2,3. */
Expand Down Expand Up @@ -328,6 +333,10 @@ during_frag( void * _ctx,

ctx->tsorig = fd_frag_meta_ts_comp( fd_tickcount() );

if( FD_UNLIKELY( in_idx==SIGN_IN_IDX ) ) {
FD_LOG_CRIT(( "signing tile send out of band fragment" ));
}

if( FD_UNLIKELY( in_idx==CONTACT_IN_IDX ) ) {
if( FD_UNLIKELY( chunk<ctx->contact_in_chunk0 || chunk>ctx->contact_in_wmark ) )
FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz,
Expand Down Expand Up @@ -398,7 +407,7 @@ during_frag( void * _ctx,
fd_shredder_init_batch( ctx->shredder, ctx->pending_batch.raw, sizeof(ulong)+ctx->pending_batch.pos, target_slot, entry_meta );

/* We sized this so it fits in one FEC set */
FD_TEST( fd_shredder_next_fec_set( ctx->shredder, ctx->shred_signing_key, out ) );
FD_TEST( fd_shredder_next_fec_set( ctx->shredder, out ) );
fd_shredder_fini_batch( ctx->shredder );

d_rcvd_join( d_rcvd_new( d_rcvd_delete( d_rcvd_leave( out->data_shred_rcvd ) ) ) );
Expand Down Expand Up @@ -672,23 +681,34 @@ privileged_init( fd_topo_t * topo,
if( FD_UNLIKELY( !strcmp( tile->shred.identity_key_path, "" ) ) )
FD_LOG_ERR(( "identity_key_path not set" ));

ctx->shred_signing_key = load_key_into_protected_memory( tile->shred.identity_key_path, /* pubkey only: */ 0 );
ctx->identity_key[ 0 ] = *(fd_pubkey_t *)load_key_into_protected_memory( tile->shred.identity_key_path, /* pubkey only: */ 1 );
}

void
fd_shred_signer( void * signer_ctx,
uchar * signature,
uchar const * merkle_root ) {
fd_remote_signer_sign_leader( signer_ctx, signature, merkle_root );
}

static void
unprivileged_init( fd_topo_t * topo,
fd_topo_tile_t * tile,
void * scratch ) {
if( FD_UNLIKELY( tile->in_cnt != 4 ||
if( FD_UNLIKELY( tile->in_cnt != 5 ||
topo->links[ tile->in_link_id[ NET_IN_IDX ] ].kind != FD_TOPO_LINK_KIND_NETMUX_TO_OUT ||
topo->links[ tile->in_link_id[ POH_IN_IDX ] ].kind != FD_TOPO_LINK_KIND_POH_TO_SHRED ||
topo->links[ tile->in_link_id[ STAKE_IN_IDX ] ].kind != FD_TOPO_LINK_KIND_STAKE_TO_OUT ||
topo->links[ tile->in_link_id[ CONTACT_IN_IDX ] ].kind != FD_TOPO_LINK_KIND_CRDS_TO_SHRED ) )
topo->links[ tile->in_link_id[ CONTACT_IN_IDX ] ].kind != FD_TOPO_LINK_KIND_CRDS_TO_SHRED ||
topo->links[ tile->in_link_id[ SIGN_IN_IDX ] ].kind != FD_TOPO_LINK_KIND_SIGN_TO_SHRED ) )
FD_LOG_ERR(( "shred tile has none or unexpected input links %lu %lu %lu",
tile->in_cnt, topo->links[ tile->in_link_id[ 0 ] ].kind, topo->links[ tile->in_link_id[ 1 ] ].kind ));

if( FD_UNLIKELY( tile->out_cnt != 1 || topo->links[ tile->out_link_id[ 0 ] ].kind != FD_TOPO_LINK_KIND_SHRED_TO_NETMUX ) )
FD_LOG_ERR(( "shred tile has none or unexpected netmux output link %lu %lu", tile->out_cnt, topo->links[ tile->out_link_id[ 0 ] ].kind ));
if( FD_UNLIKELY( tile->out_cnt != 2 ||
topo->links[ tile->out_link_id[ NET_OUT_IDX ] ].kind != FD_TOPO_LINK_KIND_SHRED_TO_NETMUX ||
topo->links[ tile->out_link_id[ SIGN_OUT_IDX ] ].kind != FD_TOPO_LINK_KIND_SHRED_TO_SIGN ) )
FD_LOG_ERR(( "shred tile has none or unexpected output links %lu %lu %lu",
tile->out_cnt, topo->links[ tile->out_link_id[ 0 ] ].kind, topo->links[ tile->out_link_id[ 1 ] ].kind ));

ulong shred_store_mcache_depth = tile->shred.depth;
if( topo->links[ tile->out_link_id_primary ].depth != shred_store_mcache_depth )
Expand Down Expand Up @@ -750,8 +770,6 @@ unprivileged_init( fd_topo_t * topo,
for( ulong j=0UL; j<FD_REEDSOL_PARITY_SHREDS_MAX; j++ ) parity_shred[ j ] = p34_base[ 2UL + j/34UL ].pkts[ j%34UL ].buffer;
}

ctx->identity_key[ 0 ] = *(fd_pubkey_t *)(ctx->shred_signing_key+32UL);

#define NONNULL( x ) (__extension__({ \
__typeof__((x)) __x = (x); \
if( FD_UNLIKELY( !__x ) ) FD_LOG_ERR(( #x " was unexpectedly NULL" )); \
Expand All @@ -770,8 +788,16 @@ unprivileged_init( fd_topo_t * topo,
FD_LOG_INFO(( "Using shred version %hu", (ushort)expected_shred_version ));

/* populate ctx */
fd_topo_link_t * sign_in = &topo->links[ tile->in_link_id[ SIGN_IN_IDX ] ];
fd_topo_link_t * sign_out = &topo->links[ tile->out_link_id[ SIGN_OUT_IDX ] ];
NONNULL( fd_remote_signer_join( fd_remote_signer_new( ctx->remote_signer_ctx,
sign_out->mcache,
sign_out->dcache,
sign_in->mcache,
sign_in->dcache ) ) );

fd_fec_set_t * resolver_sets = fec_sets + (shred_store_mcache_depth+1UL)/2UL + 1UL;
ctx->shredder = NONNULL( fd_shredder_join ( fd_shredder_new ( _shredder, ctx->identity_key->uc, (ushort)expected_shred_version ) ) );
ctx->shredder = NONNULL( fd_shredder_join ( fd_shredder_new ( _shredder, fd_shred_signer, ctx->remote_signer_ctx, (ushort)expected_shred_version ) ) );
ctx->resolver = NONNULL( fd_fec_resolver_join ( fd_fec_resolver_new ( _resolver, tile->shred.fec_resolver_depth, 1UL,
(shred_store_mcache_depth+3UL)/2UL,
128UL * tile->shred.fec_resolver_depth, resolver_sets ) ) );
Expand Down
207 changes: 207 additions & 0 deletions src/app/fdctl/run/tiles/fd_sign.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,207 @@
#include "tiles.h"

#include "generated/sign_seccomp.h"

#include "../../../../disco/keyguard/fd_keyguard.h"

#define MAX_IN (32UL)

/* fd_sign_in_ctx_t is a context object for each in (producer) mcache
connected to the sign tile. */

typedef struct {
ulong seq;
fd_frag_meta_t * mcache;
uchar * data;
} fd_sign_out_ctx_t;

typedef struct {
uchar _data[ FD_KEYGUARD_SIGN_REQ_MTU ];

ulong in_kind [ MAX_IN ];
uchar * in_data[ MAX_IN ];

fd_sign_out_ctx_t out[ MAX_IN ];

fd_sha512_t sha512 [ 1 ];

uchar const * public_key;
uchar const * private_key;
} fd_sign_ctx_t;

FD_FN_CONST static inline ulong
scratch_align( void ) {
return alignof( fd_sign_ctx_t );
}

FD_FN_PURE static inline ulong
scratch_footprint( fd_topo_tile_t * tile ) {
(void)tile;
ulong l = FD_LAYOUT_INIT;
l = FD_LAYOUT_APPEND( l, alignof( fd_sign_ctx_t ), sizeof( fd_sign_ctx_t ) );
return FD_LAYOUT_FINI( l, scratch_align() );
}

FD_FN_CONST static inline void *
mux_ctx( void * scratch ) {
return (void*)fd_ulong_align_up( (ulong)scratch, alignof( fd_sign_ctx_t ) );
}

/* during_frag is called between pairs for sequence number checks, as
we are reading incoming frags. We don't actually need to copy the
fragment here, see fd_dedup.c for why we do this.*/

static inline void
during_frag( void * _ctx,
ulong in_idx,
ulong seq,
ulong sig,
ulong chunk,
ulong sz,
int * opt_filter ) {
(void)seq;
(void)sig;
(void)chunk;
(void)sz;
(void)opt_filter;

fd_sign_ctx_t * ctx = (fd_sign_ctx_t *)_ctx;
FD_TEST( in_idx<MAX_IN );

switch( ctx->in_kind[ in_idx ] ) {
case FD_TOPO_LINK_KIND_SHRED_TO_SIGN:
fd_memcpy( ctx->_data, ctx->in_data[ in_idx ], 32UL );
break;
default:
FD_LOG_CRIT(( "unexpected link kind %lu", ctx->in_kind[ in_idx ] ));
}
}

static inline void
after_frag( void * _ctx,
ulong in_idx,
ulong seq,
ulong * opt_sig,
ulong * opt_chunk,
ulong * opt_sz,
ulong * opt_tsorig,
int * opt_filter,
fd_mux_context_t * mux ) {
(void)seq;
(void)opt_sig;
(void)opt_chunk;
(void)opt_sz;
(void)opt_tsorig;
(void)opt_filter;
(void)mux;

fd_sign_ctx_t * ctx = (fd_sign_ctx_t *)_ctx;

FD_TEST( in_idx<MAX_IN );

switch( ctx->in_kind[ in_idx ] ) {
case FD_TOPO_LINK_KIND_SHRED_TO_SIGN: {
if( FD_UNLIKELY( !fd_keyguard_payload_authorize( ctx->_data, 32UL, FD_KEYGUARD_ROLE_LEADER ) ) ) {
FD_LOG_EMERG(( "fd_keyguard_payload_authorize failed" ));
}
fd_ed25519_sign( ctx->out[ in_idx ].data, ctx->_data, 32UL, ctx->public_key, ctx->private_key, ctx->sha512 );
break;
}
default:
FD_LOG_CRIT(( "unexpected link kind %lu", ctx->in_kind[ in_idx ] ));
}

fd_mcache_publish( ctx->out[ in_idx ].mcache, 128UL, ctx->out[ in_idx ].seq, 0UL, 0UL, 0UL, 0UL, 0UL, 0UL );
ctx->out[ in_idx ].seq = fd_seq_inc( ctx->out[ in_idx ].seq, 1UL );
}

static void
privileged_init( fd_topo_t * topo,
fd_topo_tile_t * tile,
void * scratch ) {
(void)topo;

FD_SCRATCH_ALLOC_INIT( l, scratch );
fd_sign_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_sign_ctx_t ), sizeof( fd_sign_ctx_t ) );

uchar const * identity_key = load_key_into_protected_memory( tile->sign.identity_key_path, /* pubkey only: */ 0 );
ctx->private_key = identity_key;
ctx->public_key = identity_key + 32UL;
}

static void
unprivileged_init( fd_topo_t * topo,
fd_topo_tile_t * tile,
void * scratch ) {
FD_SCRATCH_ALLOC_INIT( l, scratch );
fd_sign_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_sign_ctx_t ), sizeof( fd_sign_ctx_t ) );
FD_TEST( fd_sha512_join( fd_sha512_new( ctx->sha512 ) ) );

FD_TEST( tile->in_cnt<=MAX_IN );
FD_TEST( tile->in_cnt==tile->out_cnt );

for( ulong i=0; i<MAX_IN; i++ ) ctx->in_kind[ i ] = ULONG_MAX;

for( ulong i=0; i<tile->in_cnt; i++ ) {
fd_topo_link_t * in_link = &topo->links[ tile->in_link_id[ i ] ];
fd_topo_link_t * out_link = &topo->links[ tile->out_link_id[ i ] ];

ctx->in_data[ i ] = in_link->dcache;
ctx->in_kind[ i ] = in_link->kind;

ctx->out[ i ].mcache = out_link->mcache;
ctx->out[ i ].data = out_link->dcache;
ctx->out[ i ].seq = 0UL;

switch( in_link->kind ) {
case FD_TOPO_LINK_KIND_SHRED_TO_SIGN:
FD_TEST( out_link->kind==FD_TOPO_LINK_KIND_SIGN_TO_SHRED );
FD_TEST( in_link->mtu==32UL );
FD_TEST( out_link->mtu==64UL );
break;
default:
FD_LOG_CRIT(( "unexpected link kind %lu", in_link->kind ));
}
}

ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, 1UL );
if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));
}

static ulong
populate_allowed_seccomp( void * scratch,
ulong out_cnt,
struct sock_filter * out ) {
(void)scratch;
populate_sock_filter_policy_sign( out_cnt, out, (uint)fd_log_private_logfile_fd() );
return sock_filter_policy_sign_instr_cnt;
}

static ulong
populate_allowed_fds( void * scratch,
ulong out_fds_cnt,
int * out_fds ) {
(void)scratch;
if( FD_UNLIKELY( out_fds_cnt < 2 ) ) FD_LOG_ERR(( "out_fds_cnt %lu", out_fds_cnt ));

ulong out_cnt = 0;
out_fds[ out_cnt++ ] = 2; /* stderr */
if( FD_LIKELY( -1!=fd_log_private_logfile_fd() ) )
out_fds[ out_cnt++ ] = fd_log_private_logfile_fd(); /* logfile */
return out_cnt;
}

fd_tile_config_t fd_tile_sign = {
.mux_flags = FD_MUX_FLAG_COPY | FD_MUX_FLAG_MANUAL_PUBLISH,
.burst = 1UL,
.mux_ctx = mux_ctx,
.mux_during_frag = during_frag,
.mux_after_frag = after_frag,
.populate_allowed_seccomp = populate_allowed_seccomp,
.populate_allowed_fds = populate_allowed_fds,
.scratch_align = scratch_align,
.scratch_footprint = scratch_footprint,
.privileged_init = privileged_init,
.unprivileged_init = unprivileged_init,
};
Loading

0 comments on commit df9ddc2

Please sign in to comment.