text_detokenizer.md

text_detokenizer

Source: emel/text/detokenizer/sm.hpp

Mermaid

stateDiagram-v2
  direction TB
  [*] --> uninitialized
  uninitialized --> binding : bind [valid_bind_] / begin_bind_
  uninitialized --> binding_error_decision : bind [invalid_bind_] / reject_bind_
  uninitialized --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  idle --> binding : bind [valid_bind_] / begin_bind_
  idle --> binding_error_decision : bind [invalid_bind_] / reject_bind_
  idle --> decoding : detokenize [valid_detokenize_] / begin_detokenize_
  idle --> detokenize_error_decision : detokenize [invalid_detokenize_] / reject_detokenize_
  done --> binding : bind [valid_bind_] / begin_bind_
  done --> binding_error_decision : bind [invalid_bind_] / reject_bind_
  done --> decoding : detokenize [valid_detokenize_] / begin_detokenize_
  done --> detokenize_error_decision : detokenize [invalid_detokenize_] / reject_detokenize_
  errored --> binding : bind [valid_bind_] / begin_bind_
  errored --> binding_error_decision : bind [invalid_bind_] / reject_bind_
  errored --> decoding : detokenize [valid_detokenize_] / begin_detokenize_
  errored --> detokenize_error_decision : detokenize [invalid_detokenize_] / reject_detokenize_
  unexpected --> binding : bind [valid_bind_] / begin_bind_
  unexpected --> binding_error_decision : bind [invalid_bind_] / reject_bind_
  unexpected --> decoding : detokenize [valid_detokenize_] / begin_detokenize_
  unexpected --> detokenize_error_decision : detokenize [invalid_detokenize_] / reject_detokenize_
  binding --> binding_error_decision : bind [always] / reject_bind_
  binding --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  binding_decision --> binding_error_decision : bind [always] / reject_bind_
  binding_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  binding_done_decision --> binding_error_decision : bind [always] / reject_bind_
  binding_done_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  binding_done_callback --> binding_error_decision : bind [always] / reject_bind_
  binding_done_callback --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  binding_error_decision --> binding_error_decision : bind [always] / reject_bind_
  binding_error_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  binding_error_callback --> binding_error_decision : bind [always] / reject_bind_
  binding_error_callback --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decoding --> binding_error_decision : bind [always] / reject_bind_
  decoding --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_token_validation --> binding_error_decision : bind [always] / reject_bind_
  decode_token_validation --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_piece_decision --> binding_error_decision : bind [always] / reject_bind_
  decode_piece_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_byte_capacity_decision --> binding_error_decision : bind [always] / reject_bind_
  decode_byte_capacity_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_byte_pending_decision --> binding_error_decision : bind [always] / reject_bind_
  decode_byte_pending_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_byte_pending_write --> binding_error_decision : bind [always] / reject_bind_
  decode_byte_pending_write --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_text_pending_decision --> binding_error_decision : bind [always] / reject_bind_
  decode_text_pending_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_text_pending_write --> binding_error_decision : bind [always] / reject_bind_
  decode_text_pending_write --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_text_write --> binding_error_decision : bind [always] / reject_bind_
  decode_text_write --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  decode_decision --> binding_error_decision : bind [always] / reject_bind_
  decode_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  detokenize_done_decision --> binding_error_decision : bind [always] / reject_bind_
  detokenize_done_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  detokenize_done_callback --> binding_error_decision : bind [always] / reject_bind_
  detokenize_done_callback --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  detokenize_error_decision --> binding_error_decision : bind [always] / reject_bind_
  detokenize_error_decision --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  detokenize_error_callback --> binding_error_decision : bind [always] / reject_bind_
  detokenize_error_callback --> detokenize_error_decision : detokenize [always] / reject_detokenize_
  binding --> binding_decision : completion_bind_ [always] / commit_bind_
  binding_decision --> binding_done_decision : completion_bind_ [bind_error_none_] / none
  binding_decision --> binding_error_decision : completion_bind_ [bind_error_invalid_request_] / none
  binding_decision --> binding_error_decision : completion_bind_ [bind_error_model_invalid_] / none
  binding_decision --> binding_error_decision : completion_bind_ [bind_error_backend_error_] / none
  binding_decision --> binding_error_decision : completion_bind_ [bind_error_internal_error_] / none
  binding_decision --> binding_error_decision : completion_bind_ [bind_error_untracked_] / none
  binding_decision --> binding_error_decision : completion_bind_ [bind_error_unknown_] / none
  binding_done_decision --> binding_done_callback : completion_bind_ [has_bind_done_callback_] / notify_bind_done_
  binding_done_decision --> idle : completion_bind_ [no_bind_done_callback_] / none
  binding_done_callback --> idle : completion_bind_ [always] / none
  binding_error_decision --> binding_error_callback : completion_bind_ [has_bind_error_callback_] / notify_bind_error_
  binding_error_decision --> errored : completion_bind_ [no_bind_error_callback_] / none
  binding_error_callback --> errored : completion_bind_ [always] / none
  decoding --> decode_token_validation : completion_detokenize_ [always] / none
  decode_token_validation --> decode_piece_decision : completion_detokenize_ [detokenize_token_in_vocab_] / none
  decode_token_validation --> detokenize_error_decision : completion_detokenize_ [detokenize_token_out_of_vocab_] / mark_model_invalid_
  decode_piece_decision --> detokenize_done_decision : completion_detokenize_ [detokenize_skip_special_piece_] / mark_done_
  decode_piece_decision --> decode_byte_capacity_decision : completion_detokenize_ [detokenize_byte_piece_] / none
  decode_piece_decision --> decode_text_pending_decision : completion_detokenize_ [detokenize_text_piece_] / none
  decode_piece_decision --> detokenize_error_decision : completion_detokenize_ [always] / mark_internal_error_
  decode_byte_capacity_decision --> decode_byte_pending_decision : completion_detokenize_ [detokenize_pending_has_capacity_for_byte_] / append_byte_piece_
  decode_byte_capacity_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_pending_no_capacity_for_byte_] / mark_invalid_pending_full_
  decode_byte_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_invalid_request_] / none
  decode_byte_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_model_invalid_] / none
  decode_byte_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_backend_error_] / none
  decode_byte_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_internal_error_] / none
  decode_byte_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_untracked_] / none
  decode_byte_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_unknown_] / none
  decode_byte_pending_decision --> decode_byte_pending_write : completion_detokenize_ [detokenize_pending_head_complete_] / write_pending_head_sequence_
  decode_byte_pending_decision --> decode_decision : completion_detokenize_ [detokenize_pending_empty_] / none
  decode_byte_pending_decision --> decode_decision : completion_detokenize_ [detokenize_pending_head_incomplete_] / none
  decode_byte_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_pending_head_invalid_] / mark_invalid_pending_sequence_
  decode_byte_pending_write --> decode_byte_pending_decision : completion_detokenize_ [always] / none
  decode_text_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_invalid_request_] / none
  decode_text_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_model_invalid_] / none
  decode_text_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_backend_error_] / none
  decode_text_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_internal_error_] / none
  decode_text_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_untracked_] / none
  decode_text_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_unknown_] / none
  decode_text_pending_decision --> decode_text_pending_write : completion_detokenize_ [detokenize_pending_head_complete_] / write_pending_head_sequence_
  decode_text_pending_decision --> decode_text_write : completion_detokenize_ [detokenize_pending_empty_] / write_text_piece_
  decode_text_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_pending_head_incomplete_] / mark_invalid_pending_not_empty_
  decode_text_pending_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_pending_head_invalid_] / mark_invalid_pending_sequence_
  decode_text_pending_write --> decode_text_pending_decision : completion_detokenize_ [always] / none
  decode_text_write --> decode_decision : completion_detokenize_ [always] / none
  decode_decision --> detokenize_done_decision : completion_detokenize_ [detokenize_error_none_] / mark_done_
  decode_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_invalid_request_] / none
  decode_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_model_invalid_] / none
  decode_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_backend_error_] / none
  decode_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_internal_error_] / none
  decode_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_untracked_] / none
  decode_decision --> detokenize_error_decision : completion_detokenize_ [detokenize_error_unknown_] / none
  detokenize_done_decision --> detokenize_done_callback : completion_detokenize_ [has_detokenize_done_callback_] / none
  detokenize_done_decision --> done : completion_detokenize_ [no_detokenize_done_callback_] / none
  detokenize_done_callback --> done : completion_detokenize_ [always] / notify_detokenize_done_
  detokenize_error_decision --> detokenize_error_callback : completion_detokenize_ [has_detokenize_error_callback_] / notify_detokenize_error_
  detokenize_error_decision --> errored : completion_detokenize_ [no_detokenize_error_callback_] / none
  detokenize_error_callback --> errored : completion_detokenize_ [always] / none
  uninitialized --> unexpected : _ [always] / on_unexpected_
  binding --> unexpected : _ [always] / on_unexpected_
  binding_decision --> unexpected : _ [always] / on_unexpected_
  binding_done_decision --> unexpected : _ [always] / on_unexpected_
  binding_done_callback --> unexpected : _ [always] / on_unexpected_
  binding_error_decision --> unexpected : _ [always] / on_unexpected_
  binding_error_callback --> unexpected : _ [always] / on_unexpected_
  idle --> unexpected : _ [always] / on_unexpected_
  decoding --> unexpected : _ [always] / on_unexpected_
  decode_token_validation --> unexpected : _ [always] / on_unexpected_
  decode_piece_decision --> unexpected : _ [always] / on_unexpected_
  decode_byte_capacity_decision --> unexpected : _ [always] / on_unexpected_
  decode_byte_pending_decision --> unexpected : _ [always] / on_unexpected_
  decode_byte_pending_write --> unexpected : _ [always] / on_unexpected_
  decode_text_pending_decision --> unexpected : _ [always] / on_unexpected_
  decode_text_pending_write --> unexpected : _ [always] / on_unexpected_
  decode_text_write --> unexpected : _ [always] / on_unexpected_
  decode_decision --> unexpected : _ [always] / on_unexpected_
  detokenize_done_decision --> unexpected : _ [always] / on_unexpected_
  detokenize_done_callback --> unexpected : _ [always] / on_unexpected_
  detokenize_error_decision --> unexpected : _ [always] / on_unexpected_
  detokenize_error_callback --> unexpected : _ [always] / on_unexpected_
  done --> unexpected : _ [always] / on_unexpected_
  errored --> unexpected : _ [always] / on_unexpected_
  unexpected --> unexpected : _ [always] / on_unexpected_

Loading

Transitions

Source	Event	Guard	Action	Target
uninitialized	bind	valid_bind>	begin_bind>	binding
uninitialized	bind	invalid_bind>	reject_bind>	binding_error_decision
uninitialized	detokenize	always	reject_detokenize>	detokenize_error_decision
idle	bind	valid_bind>	begin_bind>	binding
idle	bind	invalid_bind>	reject_bind>	binding_error_decision
idle	detokenize	valid_detokenize>	begin_detokenize>	decoding
idle	detokenize	invalid_detokenize>	reject_detokenize>	detokenize_error_decision
done	bind	valid_bind>	begin_bind>	binding
done	bind	invalid_bind>	reject_bind>	binding_error_decision
done	detokenize	valid_detokenize>	begin_detokenize>	decoding
done	detokenize	invalid_detokenize>	reject_detokenize>	detokenize_error_decision
errored	bind	valid_bind>	begin_bind>	binding
errored	bind	invalid_bind>	reject_bind>	binding_error_decision
errored	detokenize	valid_detokenize>	begin_detokenize>	decoding
errored	detokenize	invalid_detokenize>	reject_detokenize>	detokenize_error_decision
unexpected	bind	valid_bind>	begin_bind>	binding
unexpected	bind	invalid_bind>	reject_bind>	binding_error_decision
unexpected	detokenize	valid_detokenize>	begin_detokenize>	decoding
unexpected	detokenize	invalid_detokenize>	reject_detokenize>	detokenize_error_decision
binding	bind	always	reject_bind>	binding_error_decision
binding	detokenize	always	reject_detokenize>	detokenize_error_decision
binding_decision	bind	always	reject_bind>	binding_error_decision
binding_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
binding_done_decision	bind	always	reject_bind>	binding_error_decision
binding_done_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
binding_done_callback	bind	always	reject_bind>	binding_error_decision
binding_done_callback	detokenize	always	reject_detokenize>	detokenize_error_decision
binding_error_decision	bind	always	reject_bind>	binding_error_decision
binding_error_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
binding_error_callback	bind	always	reject_bind>	binding_error_decision
binding_error_callback	detokenize	always	reject_detokenize>	detokenize_error_decision
decoding	bind	always	reject_bind>	binding_error_decision
decoding	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_token_validation	bind	always	reject_bind>	binding_error_decision
decode_token_validation	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_piece_decision	bind	always	reject_bind>	binding_error_decision
decode_piece_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_byte_capacity_decision	bind	always	reject_bind>	binding_error_decision
decode_byte_capacity_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_byte_pending_decision	bind	always	reject_bind>	binding_error_decision
decode_byte_pending_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_byte_pending_write	bind	always	reject_bind>	binding_error_decision
decode_byte_pending_write	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_text_pending_decision	bind	always	reject_bind>	binding_error_decision
decode_text_pending_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_text_pending_write	bind	always	reject_bind>	binding_error_decision
decode_text_pending_write	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_text_write	bind	always	reject_bind>	binding_error_decision
decode_text_write	detokenize	always	reject_detokenize>	detokenize_error_decision
decode_decision	bind	always	reject_bind>	binding_error_decision
decode_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
detokenize_done_decision	bind	always	reject_bind>	binding_error_decision
detokenize_done_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
detokenize_done_callback	bind	always	reject_bind>	binding_error_decision
detokenize_done_callback	detokenize	always	reject_detokenize>	detokenize_error_decision
detokenize_error_decision	bind	always	reject_bind>	binding_error_decision
detokenize_error_decision	detokenize	always	reject_detokenize>	detokenize_error_decision
detokenize_error_callback	bind	always	reject_bind>	binding_error_decision
detokenize_error_callback	detokenize	always	reject_detokenize>	detokenize_error_decision
binding	completion<bind>	always	commit_bind>	binding_decision
binding_decision	completion<bind>	bind_error_none>	none	binding_done_decision
binding_decision	completion<bind>	bind_error_invalid_request>	none	binding_error_decision
binding_decision	completion<bind>	bind_error_model_invalid>	none	binding_error_decision
binding_decision	completion<bind>	bind_error_backend_error>	none	binding_error_decision
binding_decision	completion<bind>	bind_error_internal_error>	none	binding_error_decision
binding_decision	completion<bind>	bind_error_untracked>	none	binding_error_decision
binding_decision	completion<bind>	bind_error_unknown>	none	binding_error_decision
binding_done_decision	completion<bind>	has_bind_done_callback>	notify_bind_done>	binding_done_callback
binding_done_decision	completion<bind>	no_bind_done_callback>	none	idle
binding_done_callback	completion<bind>	always	none	idle
binding_error_decision	completion<bind>	has_bind_error_callback>	notify_bind_error>	binding_error_callback
binding_error_decision	completion<bind>	no_bind_error_callback>	none	errored
binding_error_callback	completion<bind>	always	none	errored
decoding	completion<detokenize>	always	none	decode_token_validation
decode_token_validation	completion<detokenize>	detokenize_token_in_vocab>	none	decode_piece_decision
decode_token_validation	completion<detokenize>	detokenize_token_out_of_vocab>	mark_model_invalid>	detokenize_error_decision
decode_piece_decision	completion<detokenize>	detokenize_skip_special_piece>	mark_done>	detokenize_done_decision
decode_piece_decision	completion<detokenize>	detokenize_byte_piece>	none	decode_byte_capacity_decision
decode_piece_decision	completion<detokenize>	detokenize_text_piece>	none	decode_text_pending_decision
decode_piece_decision	completion<detokenize>	always	mark_internal_error>	detokenize_error_decision
decode_byte_capacity_decision	completion<detokenize>	detokenize_pending_has_capacity_for_byte>	append_byte_piece>	decode_byte_pending_decision
decode_byte_capacity_decision	completion<detokenize>	detokenize_pending_no_capacity_for_byte>	mark_invalid_pending_full>	detokenize_error_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_error_invalid_request>	none	detokenize_error_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_error_model_invalid>	none	detokenize_error_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_error_backend_error>	none	detokenize_error_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_error_internal_error>	none	detokenize_error_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_error_untracked>	none	detokenize_error_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_error_unknown>	none	detokenize_error_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_pending_head_complete>	write_pending_head_sequence>	decode_byte_pending_write
decode_byte_pending_decision	completion<detokenize>	detokenize_pending_empty>	none	decode_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_pending_head_incomplete>	none	decode_decision
decode_byte_pending_decision	completion<detokenize>	detokenize_pending_head_invalid>	mark_invalid_pending_sequence>	detokenize_error_decision
decode_byte_pending_write	completion<detokenize>	always	none	decode_byte_pending_decision
decode_text_pending_decision	completion<detokenize>	detokenize_error_invalid_request>	none	detokenize_error_decision
decode_text_pending_decision	completion<detokenize>	detokenize_error_model_invalid>	none	detokenize_error_decision
decode_text_pending_decision	completion<detokenize>	detokenize_error_backend_error>	none	detokenize_error_decision
decode_text_pending_decision	completion<detokenize>	detokenize_error_internal_error>	none	detokenize_error_decision
decode_text_pending_decision	completion<detokenize>	detokenize_error_untracked>	none	detokenize_error_decision
decode_text_pending_decision	completion<detokenize>	detokenize_error_unknown>	none	detokenize_error_decision
decode_text_pending_decision	completion<detokenize>	detokenize_pending_head_complete>	write_pending_head_sequence>	decode_text_pending_write
decode_text_pending_decision	completion<detokenize>	detokenize_pending_empty>	write_text_piece>	decode_text_write
decode_text_pending_decision	completion<detokenize>	detokenize_pending_head_incomplete>	mark_invalid_pending_not_empty>	detokenize_error_decision
decode_text_pending_decision	completion<detokenize>	detokenize_pending_head_invalid>	mark_invalid_pending_sequence>	detokenize_error_decision
decode_text_pending_write	completion<detokenize>	always	none	decode_text_pending_decision
decode_text_write	completion<detokenize>	always	none	decode_decision
decode_decision	completion<detokenize>	detokenize_error_none>	mark_done>	detokenize_done_decision
decode_decision	completion<detokenize>	detokenize_error_invalid_request>	none	detokenize_error_decision
decode_decision	completion<detokenize>	detokenize_error_model_invalid>	none	detokenize_error_decision
decode_decision	completion<detokenize>	detokenize_error_backend_error>	none	detokenize_error_decision
decode_decision	completion<detokenize>	detokenize_error_internal_error>	none	detokenize_error_decision
decode_decision	completion<detokenize>	detokenize_error_untracked>	none	detokenize_error_decision
decode_decision	completion<detokenize>	detokenize_error_unknown>	none	detokenize_error_decision
detokenize_done_decision	completion<detokenize>	has_detokenize_done_callback>	none	detokenize_done_callback
detokenize_done_decision	completion<detokenize>	no_detokenize_done_callback>	none	done
detokenize_done_callback	completion<detokenize>	always	notify_detokenize_done>	done
detokenize_error_decision	completion<detokenize>	has_detokenize_error_callback>	notify_detokenize_error>	detokenize_error_callback
detokenize_error_decision	completion<detokenize>	no_detokenize_error_callback>	none	errored
detokenize_error_callback	completion<detokenize>	always	none	errored
uninitialized	_	always	on_unexpected>	unexpected
binding	_	always	on_unexpected>	unexpected
binding_decision	_	always	on_unexpected>	unexpected
binding_done_decision	_	always	on_unexpected>	unexpected
binding_done_callback	_	always	on_unexpected>	unexpected
binding_error_decision	_	always	on_unexpected>	unexpected
binding_error_callback	_	always	on_unexpected>	unexpected
idle	_	always	on_unexpected>	unexpected
decoding	_	always	on_unexpected>	unexpected
decode_token_validation	_	always	on_unexpected>	unexpected
decode_piece_decision	_	always	on_unexpected>	unexpected
decode_byte_capacity_decision	_	always	on_unexpected>	unexpected
decode_byte_pending_decision	_	always	on_unexpected>	unexpected
decode_byte_pending_write	_	always	on_unexpected>	unexpected
decode_text_pending_decision	_	always	on_unexpected>	unexpected
decode_text_pending_write	_	always	on_unexpected>	unexpected
decode_text_write	_	always	on_unexpected>	unexpected
decode_decision	_	always	on_unexpected>	unexpected
detokenize_done_decision	_	always	on_unexpected>	unexpected
detokenize_done_callback	_	always	on_unexpected>	unexpected
detokenize_error_decision	_	always	on_unexpected>	unexpected
detokenize_error_callback	_	always	on_unexpected>	unexpected
done	_	always	on_unexpected>	unexpected
errored	_	always	on_unexpected>	unexpected
unexpected	_	always	on_unexpected>	unexpected