diff --git a/cpp/csp/cppnodes/baselibimpl.cpp b/cpp/csp/cppnodes/baselibimpl.cpp index 52a5537d9..0c9179ec9 100644 --- a/cpp/csp/cppnodes/baselibimpl.cpp +++ b/cpp/csp/cppnodes/baselibimpl.cpp @@ -705,6 +705,9 @@ DECLARE_CPPNODE( struct_fromts ) ); } + if( unlikely( !out.get() -> validate() ) ) + CSP_THROW( ValueError, "Struct " << cls.value() -> name() << " is not valid; required fields " << out -> formatAllUnsetStrictFields() << " did not tick" ); + CSP_OUTPUT( std::move( out ) ); } @@ -758,6 +761,9 @@ DECLARE_CPPNODE( struct_collectts ) } ); } + + if( unlikely( !out.get() -> validate() ) ) + CSP_THROW( ValueError, "Struct " << cls.value() -> name() << " is not valid; required fields " << out -> formatAllUnsetStrictFields() << " did not tick" ); CSP_OUTPUT( std::move( out ) ); } diff --git a/cpp/csp/engine/InputAdapter.h b/cpp/csp/engine/InputAdapter.h index f992d6ff2..e1e9d0a0e 100644 --- a/cpp/csp/engine/InputAdapter.h +++ b/cpp/csp/engine/InputAdapter.h @@ -4,6 +4,7 @@ #include #include #include +#include #include namespace csp @@ -22,6 +23,8 @@ class InputAdapter : public TimeSeriesProvider, public EngineOwned virtual void start( DateTime start, DateTime end ) {} virtual void stop() {} + virtual const char * name() const { return "InputAdapter"; } + template< typename T > void outputTickTyped( DateTime timestamp, const T & value ) { outputTickTyped( rootEngine() -> cycleCount(), timestamp, value ); @@ -55,6 +58,13 @@ class InputAdapter : public TimeSeriesProvider, public EngineOwned template bool InputAdapter::consumeTick( const T & value ) { + if constexpr( CspType::Type::fromCType::type == CspType::TypeTraits::STRUCT ) + { + if( unlikely( !( value -> validate() ) ) ) + CSP_THROW( ValueError, "Struct " << value -> meta() -> name() << " from adapter type " << name() + << " is not valid; required fields " << value -> formatAllUnsetStrictFields() << " were not set on init" ); + } + switch( pushMode() ) { case PushMode::LAST_VALUE: diff --git a/cpp/csp/engine/Struct.cpp b/cpp/csp/engine/Struct.cpp index 42830357e..b1def5c94 100644 --- a/cpp/csp/engine/Struct.cpp +++ b/cpp/csp/engine/Struct.cpp @@ -1,20 +1,26 @@ #include #include #include +#include +#include +#include namespace csp { StructField::StructField( CspTypePtr type, const std::string & fieldname, - size_t size, size_t alignment ) : + size_t size, size_t alignment, bool isOptional ) : m_fieldname( fieldname ), m_offset( 0 ), m_size( size ), m_alignment( alignment ), m_maskOffset( 0 ), m_maskBit( 0 ), + m_noneMaskBit( 0 ), m_maskBitMask( 0 ), - m_type( type ) + m_noneMaskBitMask( 0 ), + m_type( type ), + m_isOptional( isOptional ) { } @@ -33,13 +39,13 @@ and adjustments required for the hidden fields */ -StructMeta::StructMeta( const std::string & name, const Fields & fields, - std::shared_ptr base ) : m_name( name ), m_base( base ), m_fields( fields ), +StructMeta::StructMeta( const std::string & name, const Fields & fields, bool isStrict, + std::shared_ptr base ) : m_name( name ), m_base( base ), m_fields( fields ), m_optionalFieldsBitMasks(), m_size( 0 ), m_partialSize( 0 ), m_partialStart( 0 ), m_nativeStart( 0 ), m_basePadding( 0 ), m_maskLoc( 0 ), m_maskSize( 0 ), m_firstPartialField( 0 ), m_firstNativePartialField( 0 ), - m_isPartialNative( true ), m_isFullyNative( true ) + m_isPartialNative( true ), m_isFullyNative( true ), m_isStrict( isStrict ) { - if( m_fields.empty() && !m_base) + if( m_fields.empty() && !m_base ) CSP_THROW( TypeError, "Struct types must define at least 1 field" ); //sort by sizes, biggest first, to get proper alignment @@ -95,25 +101,64 @@ StructMeta::StructMeta( const std::string & name, const Fields & fields, //Setup masking bits for our fields //NOTE we can be more efficient by sticking masks into any potential alignment gaps, dont want to spend time on it //at this point - m_maskSize = !m_fields.empty() ? 1 + ( ( m_fields.size() - 1 ) / 8 ) : 0; + + size_t optionalFieldCount = std::count_if( m_fields.begin(), m_fields.end(), []( const auto & f ) { return f -> isOptional(); } ); + + m_maskSize = !m_fields.empty() ? 1 + ( ( m_fields.size() + optionalFieldCount - 1 ) / 8 ) : 0; m_size = offset + m_maskSize; m_partialSize = m_size - baseSize; m_maskLoc = m_size - m_maskSize; + + uint8_t numRemainingBits = ( m_fields.size() + optionalFieldCount ) % 8; + m_lastByteMask = ( 1u << numRemainingBits ) - 1; size_t maskLoc = m_maskLoc; uint8_t maskBit = 0; - for( auto & f : m_fields ) + + // Set optional fields first so that their 2-bits never cross a byte boundary + // Put both the set bits and none bits in the same vector to avoid fragmentation + m_optionalFieldsBitMasks.resize( 2 * m_maskSize ); + for( size_t i = 0; i < m_fields.size(); ++i ) + { + auto & f = m_fields[ i ]; + if( f -> isOptional() ) + { + f -> setMaskOffset( maskLoc, maskBit ); + optionalSetBitMask( maskLoc - m_maskLoc ) |= ( 1 << maskBit ); // set bits for this byte + optionalNoneBitMask( maskLoc - m_maskLoc ) |= ( 1 << ++maskBit ); // none bits for this byte + if( ++maskBit == 8 ) + { + maskBit = 0; + ++maskLoc; + } + } + } + + for( size_t i = 0; i < m_fields.size(); ++i ) { - f -> setMaskOffset( maskLoc, maskBit ); - if( ++maskBit == 8 ) + auto & f = m_fields[ i ]; + if( !f -> isOptional() ) { - maskBit = 0; - ++maskLoc; + f -> setMaskOffset( maskLoc, maskBit ); + if( ++maskBit == 8 ) + { + maskBit = 0; + ++maskLoc; + } } } if( m_base ) { + // The complete inheritance hierarchy must agree on strict/non-strict + if( m_isStrict != m_base -> isStrict() ) + { + CSP_THROW( ValueError, + "Struct " << m_name << " was declared " << ( m_isStrict ? "strict" : "non-strict" ) << " but derives from " + << m_base -> name() << " which is " << ( m_base -> isStrict() ? "strict" : "non-strict" ) + ); + } + m_fields.insert( m_fields.begin(), m_base -> m_fields.begin(), m_base -> m_fields.end() ); m_fieldnames.insert( m_fieldnames.begin(), m_base -> m_fieldnames.begin(), m_base -> m_fieldnames.end() ); @@ -453,23 +498,60 @@ void StructMeta::clear( Struct * s ) const m_base -> clear( s ); } +std::string StructMeta::formatAllUnsetStrictFields( const Struct * s ) const +{ + bool first = true; + std::stringstream ss; + ss << "["; + + for( size_t i = 0; i < m_fields.size(); ++i ) + { + const auto & f = m_fields[ i ]; + if( !f -> isSet( s ) && !f -> isNone( s ) ) + { + if( !first ) + ss << ", "; + else + first = false; + ss << f -> fieldname(); + } + } + ss << "]"; + return ss.str(); +} + bool StructMeta::allFieldsSet( const Struct * s ) const { - size_t numLocalFields = m_fields.size() - m_firstPartialField; - uint8_t numRemainingBits = numLocalFields % 8; + /* + We can use bit operations to validate the struct. + 1. Let M1 be the bitmask with 1s that align with the set bit of optional fields and + 2. Let M2 be the bitmask with 1s that align with the none bit of optional fields. + -- Both M1 and M2 are computed trivially when we create the meta. + 3. Let M1* = M1 & mask. M1* now is the bitmask of all set optional fields. + 4. Similarly, let M2* = M2 & mask, such that M2* is the bitmask of all none optional fields. + 5. Let M3 = mask | (M1* << 1) | (M2* >> 1). Since the shifted set/none bitsets will fill in that optional fields other bit, + the struct can validate iff M3 is all 1s. + + Notes: + 1) We do this on a byte by byte basis currently. If we add 32/64 bit padding to the struct mask, we could do this as one single step for most structs. + 2) There is an edge condition if a) the set bit of an optional field is the last bit of a byte or b) the none bit of an optional field is the first bit of a byte. + So, when we create the meta, we ensure this never happens by being smart about the ordering of fields in the mask. + */ const uint8_t * m = reinterpret_cast( s ) + m_maskLoc; - const uint8_t * e = m + m_maskSize - bool( numRemainingBits ); - for( ; m < e; ++m ) + const uint8_t * e = m + m_maskSize - bool( m_lastByteMask ); + + size_t i = 0; + for( ; m < e; ++m, ++i ) { - if( *m != 0xFF ) + if( ( *m | ( ( *m & optionalSetBitMask( i ) ) << 1 ) | ( ( *m & optionalNoneBitMask( i ) ) >> 1 ) ) != 0xFF ) return false; } - if( numRemainingBits > 0 ) + if( m_lastByteMask ) { - uint8_t bitmask = ( 1u << numRemainingBits ) - 1; - if( ( *m & bitmask ) != bitmask ) + uint8_t masked = *m & m_lastByteMask; + if( ( masked | ( ( ( masked & optionalSetBitMask( i ) ) << 1 ) & m_lastByteMask ) | ( ( masked & optionalNoneBitMask( i ) ) >> 1 ) ) != m_lastByteMask ) return false; } @@ -494,6 +576,11 @@ void StructMeta::destroy( Struct * s ) const m_base -> destroy( s ); } +[[nodiscard]] bool StructMeta::validate( const Struct * s ) const +{ + return !isStrict() || allFieldsSet( s ); +} + Struct::Struct( const std::shared_ptr & meta ) { //Initialize meta shared_ptr diff --git a/cpp/csp/engine/Struct.h b/cpp/csp/engine/Struct.h index 64b51ecae..54ed071a4 100644 --- a/cpp/csp/engine/Struct.h +++ b/cpp/csp/engine/Struct.h @@ -22,6 +22,8 @@ class StructField { public: + static constexpr uint8_t BITS_PER_BYTE = 8; + virtual ~StructField() {} const std::string & fieldname() const { return m_fieldname; } @@ -29,20 +31,30 @@ class StructField size_t offset() const { return m_offset; } //offset to start of field's memory from start of struct mem size_t size() const { return m_size; } //size of field in bytes size_t alignment() const { return m_alignment; } //alignment of the field - size_t maskOffset() const { return m_maskOffset; } //offset to location of the mask byte fo this field from start of struct mem - uint8_t maskBit() const { return m_maskBit; } //bit within mask byte associated with this field + size_t maskOffset() const { return m_maskOffset; } //offset to location of the mask byte of this field from start of struct mem. Note that set/none bits always live on same byte + uint8_t maskBit() const { return m_maskBit; } //bit within mask byte associated with this field. Just used for debugging uint8_t maskBitMask() const { return m_maskBitMask; } //same as maskBit but as a mask ( 1 << bit + uint8_t noneMaskBit() const { return m_noneMaskBit; } + uint8_t noneMaskBitMask() const { return m_noneMaskBitMask; } bool isNative() const { return m_type -> type() <= CspType::Type::MAX_NATIVE_TYPE; } + bool isOptional() const { return m_isOptional; } void setOffset( size_t off ) { m_offset = off; } - void setMaskOffset( size_t off, uint8_t bit ) + void setMaskOffset( size_t off, uint8_t bit ) { - CSP_ASSERT( bit < 8 ); + CSP_ASSERT( bit < BITS_PER_BYTE ); m_maskOffset = off; m_maskBit = bit; - m_maskBitMask = 1 << bit; + m_maskBitMask = 1 << m_maskBit; + if( m_isOptional ) + { + // None mask bit is always adjacent and on the same byte as the set bit + CSP_ASSERT( bit + 1 < BITS_PER_BYTE ); + m_noneMaskBit = bit + 1; + m_noneMaskBitMask = 1 << m_noneMaskBit; + } } bool isSet( const Struct * s ) const @@ -51,6 +63,20 @@ class StructField return (*m ) & m_maskBitMask; } + bool isNone( const Struct * s ) const + { + const uint8_t * m = reinterpret_cast( s ) + m_maskOffset; + return ( *m ) & m_noneMaskBitMask; + } + + void setNone( Struct * s ) const + { + CSP_ASSERT( m_isOptional ); + + uint8_t * m = reinterpret_cast( s ) + m_maskOffset; + (*m) |= m_noneMaskBitMask; + } + //copy methods need not deal with mask set/unset, only copy values virtual void copyFrom( const Struct * src, Struct * dest ) const = 0; @@ -75,12 +101,14 @@ class StructField protected: StructField( CspTypePtr type, const std::string & fieldname, - size_t size, size_t alignment ); + size_t size, size_t alignment, bool optional ); void setIsSet( Struct * s ) const { uint8_t * m = reinterpret_cast( s ) + m_maskOffset; (*m) |= m_maskBitMask; + + clearIsNone( s ); // no-op if not an optional field, as m_noneMaskBitMask is initialized to 0 in the constructor } const void * valuePtr( const Struct * s ) const @@ -99,6 +127,12 @@ class StructField (*m) &= ~m_maskBitMask; } + void clearIsNone( Struct * s ) const + { + uint8_t * m = reinterpret_cast( s ) + m_maskOffset; + (*m) &= ~m_noneMaskBitMask; + } + private: std::string m_fieldname; size_t m_offset; @@ -106,8 +140,11 @@ class StructField const size_t m_alignment; size_t m_maskOffset; uint8_t m_maskBit; + uint8_t m_noneMaskBit; uint8_t m_maskBitMask; + uint8_t m_noneMaskBitMask; CspTypePtr m_type; + bool m_isOptional; }; using StructFieldPtr = std::shared_ptr; @@ -120,7 +157,7 @@ class NativeStructField : public StructField public: NativeStructField() {} - NativeStructField( const std::string & fieldname ) : NativeStructField( CspType::fromCType::type(), fieldname ) + NativeStructField( const std::string & fieldname, bool optional ) : NativeStructField( CspType::fromCType::type(), fieldname, optional ) { } @@ -157,7 +194,8 @@ class NativeStructField : public StructField } protected: - NativeStructField( CspTypePtr type, const std::string & fieldname ) : StructField( type, fieldname, sizeof( T ), alignof( T ) ) + NativeStructField( CspTypePtr type, const std::string & fieldname, bool optional ) + : StructField( type, fieldname, sizeof( T ), alignof( T ), optional ) {} }; @@ -179,7 +217,7 @@ using TimeStructField = NativeStructField