Chat: STT integration (#32505)

Co-authored-by: marker dao ® <youdontknow@marker-dao.eth>

Author: marker-dao

packages/devextreme/js/__internal/ui/chat/chat.ts

@@ -16,10 +16,11 @@ import type {
   MessageEnteredEvent,
   MessageUpdatedEvent,
   MessageUpdatingEvent,
-  Properties,
+  Properties as ChatProperties,
   TypingEndEvent,
   TypingStartEvent,
 } from '@js/ui/chat';
+import type { Properties as SpeechToTextProperties } from '@js/ui/speech_to_text';
 import { invokeConditionally } from '@ts/core/utils/conditional_invoke';
 import type { OptionChanged } from '@ts/core/widget/types';
 import Widget from '@ts/core/widget/widget';
@@ -40,6 +41,11 @@ import type {
 import MessageList from '@ts/ui/chat/messagelist';
 import type { DataChange } from '@ts/ui/collection/collection_widget.base';
 
+type Properties = ChatProperties & {
+  speechToTextEnabled?: boolean;
+  speechToTextOptions?: SpeechToTextProperties;
+};
+
 const CHAT_CLASS = 'dx-chat';
 const TEXTEDITOR_INPUT_CLASS = 'dx-texteditor-input';
 
@@ -101,6 +107,8 @@ class Chat extends Widget<Properties> {
       showDayHeaders: true,
       showMessageTimestamp: true,
       showUserName: true,
+      speechToTextEnabled: false,
+      speechToTextOptions: undefined,
       typingUsers: [],
       user: { id: new Guid().toString() },
       onMessageDeleted: undefined,
@@ -464,6 +472,8 @@ class Chat extends Widget<Properties> {
       hoverStateEnabled,
       // @ts-expect-error wait for .d.ts
       inputFieldText,
+      speechToTextEnabled,
+      speechToTextOptions,
     } = this.option();
 
     const $messageBox = $('<div>');
@@ -476,6 +486,8 @@ class Chat extends Widget<Properties> {
       focusStateEnabled,
       hoverStateEnabled,
       text: inputFieldText,
+      speechToTextEnabled,
+      speechToTextOptions,
       onMessageEntered: (e) => {
         this._messageEnteredHandler(e);
       },
@@ -644,6 +656,8 @@ class Chat extends Widget<Properties> {
       case 'hoverStateEnabled':
         this._messageBox.option(name, value);
         break;
+      case 'speechToTextEnabled':
+      case 'speechToTextOptions':
       case 'fileUploaderOptions':
         this._messageBox.option(fullName, value);
         break;

packages/devextreme/js/__internal/ui/chat/message_box/chat_text_area.ts

@@ -11,19 +11,53 @@ import type {
 } from '@js/ui/button';
 import type Button from '@js/ui/button';
 import type { Attachment } from '@js/ui/chat';
-import type { UploadedEvent, UploadStartedEvent, ValueChangedEvent } from '@js/ui/file_uploader';
+import type {
+  UploadedEvent,
+  UploadStartedEvent,
+  ValueChangedEvent,
+} from '@js/ui/file_uploader';
+import type dxSpeechToText from '@js/ui/speech_to_text';
+import type {
+  EndEvent,
+  InitializedEvent as InitializedSTTEvent,
+  Properties as SpeechToTextProperties,
+  ResultEvent,
+  StartClickEvent,
+} from '@js/ui/speech_to_text';
 import { current, isMaterial } from '@js/ui/themes';
 import type { Item as ToolbarItem } from '@js/ui/toolbar';
 import Toolbar from '@js/ui/toolbar';
 import type { OptionChanged } from '@ts/core/widget/types';
 import type { SupportedKeys } from '@ts/core/widget/widget';
 import Widget from '@ts/core/widget/widget';
 import FileUploader from '@ts/ui/file_uploader/file_uploader';
-import type { CancelButtonClickEvent, FileValidationErrorEvent, Properties as FileUploaderProperties } from '@ts/ui/file_uploader/file_uploader.types';
+import type {
+  CancelButtonClickEvent,
+  FileValidationErrorEvent,
+  Properties as FileUploaderProperties,
+} from '@ts/ui/file_uploader/file_uploader.types';
 import Informer from '@ts/ui/informer/informer';
 import type { TextAreaProperties } from '@ts/ui/m_text_area';
 import TextArea from '@ts/ui/m_text_area';
 
+type EnterKeyEvent = NativeEventInfo<ChatTextArea, KeyboardEvent>;
+
+export type SendEvent = ClickEvent | EnterKeyEvent;
+
+type FileToSend = Attachment & {
+  readyToSend: boolean;
+};
+
+export type Properties = TextAreaProperties & {
+  fileUploaderOptions?: FileUploaderProperties;
+
+  speechToTextEnabled?: boolean;
+
+  speechToTextOptions?: SpeechToTextProperties;
+
+  onSend?: (e: SendEvent) => void;
+};
+
 const CHAT_TEXT_AREA_ATTACHMENTS = 'dx-chat-textarea-attachments';
 export const CHAT_TEXT_AREA_ATTACH_BUTTON = 'dx-chat-textarea-attach-button';
 
@@ -38,23 +72,15 @@ const ERRORS = {
   fileLimit: messageLocalization.format('dxChat-fileLimitReachedWarning', MAX_ATTACHMENTS_COUNT),
 };
 
-const isMobile = (): boolean => devices.current().deviceType !== 'desktop';
-
-export const DEFAULT_ALLOWED_FILE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.pdf', '.docx', '.xlsx', '.pptx', '.txt', '.rtf', '.csv', '.md'];
-
-type EnterKeyEvent = NativeEventInfo<ChatTextArea, KeyboardEvent>;
-
-export type SendEvent = ClickEvent | EnterKeyEvent;
-
-type FileToSend = Attachment & {
-  readyToSend: boolean;
+const STT_INITIAL_STATE = {
+  stylingMode: 'text' as const,
+  type: 'normal',
+  stopIcon: 'micfilled',
 };
 
-export type Properties = TextAreaProperties & {
-  fileUploaderOptions?: FileUploaderProperties;
+const isMobile = (): boolean => devices.current().deviceType !== 'desktop';
 
-  onSend?: (e: SendEvent) => void;
-};
+export const DEFAULT_ALLOWED_FILE_EXTENSIONS = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.pdf', '.docx', '.xlsx', '.pptx', '.txt', '.rtf', '.csv', '.md'];
 
 class ChatTextArea extends TextArea<Properties> {
   _informerTimeoutId?: ReturnType<typeof setTimeout> | undefined;
@@ -71,10 +97,14 @@ class ChatTextArea extends TextArea<Properties> {
 
   _filesToSend?: Map<File, FileToSend>;
 
+  _initialInputText?: string;
+
   _attachButton?: Button;
 
   _sendButton?: Button;
 
+  _speechToTextButton?: dxSpeechToText;
+
   _sendAction?: (e: Partial<SendEvent>) => void;
 
   getAttachments(): Attachment[] | undefined {
@@ -90,11 +120,13 @@ class ChatTextArea extends TextArea<Properties> {
   _getDefaultOptions(): Properties {
     return {
       ...super._getDefaultOptions(),
-      stylingMode: 'outlined',
-      placeholder: messageLocalization.format('dxChat-textareaPlaceholder'),
       autoResizeEnabled: true,
-      valueChangeEvent: 'input',
       maxHeight: '53.86em',
+      placeholder: messageLocalization.format('dxChat-textareaPlaceholder'),
+      speechToTextEnabled: false,
+      speechToTextOptions: undefined,
+      stylingMode: 'outlined',
+      valueChangeEvent: 'input',
     };
   }
 
@@ -136,6 +168,8 @@ class ChatTextArea extends TextArea<Properties> {
   }
 
   _init(): void {
+    this._initialInputText = '';
+
     super._init();
 
     this._createSendAction();
@@ -208,16 +242,20 @@ class ChatTextArea extends TextArea<Properties> {
   }
 
   _getToolbarItems(): ToolbarItem[] {
-    const { fileUploaderOptions } = this.option();
+    const { fileUploaderOptions, speechToTextEnabled } = this.option();
 
-    const items = [
-      this._getSendButtonConfig(),
-    ];
+    const items: ToolbarItem[] = [];
 
     if (fileUploaderOptions) {
       items.push(this._getAttachButtonConfig());
     }
 
+    if (speechToTextEnabled) {
+      items.push(this._getSpeechToTextButtonConfig());
+    }
+
+    items.push(this._getSendButtonConfig());
+
     return items;
   }
 
@@ -247,6 +285,54 @@ class ChatTextArea extends TextArea<Properties> {
     return configuration;
   }
 
+  _getSpeechToTextButtonOptions(): SpeechToTextProperties {
+    const {
+      activeStateEnabled,
+      focusStateEnabled,
+      hoverStateEnabled,
+      speechToTextOptions,
+    } = this.option();
+
+    const options = {
+      activeStateEnabled,
+      focusStateEnabled,
+      hoverStateEnabled,
+      ...speechToTextOptions,
+      ...STT_INITIAL_STATE,
+      onEnd: (e: EndEvent): void => {
+        this._initialInputText = '';
+
+        speechToTextOptions?.onEnd?.(e);
+      },
+      onInitialized: (e: InitializedSTTEvent): void => {
+        this._speechToTextButton = e.component;
+
+        speechToTextOptions?.onInitialized?.(e);
+      },
+      onResult: (e: ResultEvent): void => this._resultHandler(e),
+      onStartClick: (e: StartClickEvent): void => {
+        const { text } = this.option();
+
+        this._initialInputText = text;
+
+        speechToTextOptions?.onStartClick?.(e);
+      },
+    };
+
+    return options;
+  }
+
+  _getSpeechToTextButtonConfig(): ToolbarItem {
+    // @ts-expect-error dxSpeechToText should be added to ToolbarItemComponent
+    const configuration = {
+      widget: 'dxSpeechToText',
+      location: 'after',
+      options: this._getSpeechToTextButtonOptions(),
+    } as ToolbarItem;
+
+    return configuration;
+  }
+
   _getSendButtonConfig(): ToolbarItem {
     const {
       activeStateEnabled,
@@ -280,6 +366,22 @@ class ChatTextArea extends TextArea<Properties> {
     return configuration;
   }
 
+  _resultHandler(e: ResultEvent): void {
+    const { speechToTextOptions } = this.option();
+
+    // @ts-expect-error SpeechRecognition API is not supported in TS
+    const speechRecognitionResult = Object.values(e.event.results)
+    // @ts-expect-error SpeechRecognition API is not supported in TS
+      .map((resultItem) => (resultItem[0].transcript as string).trim())
+      .join(' ');
+
+    const result = `${this._initialInputText} ${speechRecognitionResult}`.trim();
+
+    this.option({ value: result });
+
+    speechToTextOptions?.onResult?.(e);
+  }
+
   _initFileUploader(): void {
     const { fileUploaderOptions } = this.option();
 
@@ -449,6 +551,7 @@ class ChatTextArea extends TextArea<Properties> {
       case 'focusStateEnabled':
       case 'hoverStateEnabled':
         this._sendButton?.option(name, value);
+        this._speechToTextButton?.option(name, value);
         break;
 
       case 'text':
@@ -464,6 +567,14 @@ class ChatTextArea extends TextArea<Properties> {
         this._handleFileUploaderOptionsChange(args);
         break;
 
+      case 'speechToTextEnabled':
+        this._toolbar?.option({ items: this._getToolbarItems() });
+        break;
+
+      case 'speechToTextOptions':
+        this._speechToTextButton?.option(this._getSpeechToTextButtonOptions());
+        break;
+
       default:
         super._optionChanged(args);
     }

packages/devextreme/js/__internal/ui/chat/message_box/message_box.ts

@@ -3,6 +3,7 @@ import $, { type dxElementWrapper } from '@js/core/renderer';
 import type { InteractionEvent } from '@js/events';
 import type { Attachment } from '@js/ui/chat';
 import type { Properties as FileUploaderProperties } from '@js/ui/file_uploader';
+import type { Properties as SpeechToTextProperties } from '@js/ui/speech_to_text';
 import type { InputEvent } from '@js/ui/text_area';
 import type { DOMComponentProperties } from '@ts/core/widget/dom_component';
 import DOMComponent from '@ts/core/widget/dom_component';
@@ -14,12 +15,6 @@ import type {
 import ChatTextArea from '@ts/ui/chat/message_box/chat_text_area';
 import EditingPreview from '@ts/ui/chat/message_box/editing_preview';
 
-export const CHAT_MESSAGEBOX_CLASS = 'dx-chat-messagebox';
-export const CHAT_MESSAGEBOX_TEXTAREA_CONTAINER_CLASS = 'dx-chat-messagebox-textarea-container';
-
-export const TYPING_END_DELAY = 2000;
-const ESCAPE_KEY = 'escape';
-
 export type MessageEnteredEvent = NativeEventInfo<MessageBox, InteractionEvent>
   & {
     text?: string;
@@ -39,6 +34,10 @@ export interface Properties extends DOMComponentProperties<MessageBox> {
 
   previewText?: string;
 
+  speechToTextEnabled?: boolean;
+
+  speechToTextOptions?: SpeechToTextProperties;
+
   text?: string;
 
   onMessageEntered?: (e: MessageEnteredEvent) => void;
@@ -52,6 +51,12 @@ export interface Properties extends DOMComponentProperties<MessageBox> {
   onMessageUpdating?: (e: { text: string }) => void;
 }
 
+export const CHAT_MESSAGEBOX_CLASS = 'dx-chat-messagebox';
+export const CHAT_MESSAGEBOX_TEXTAREA_CONTAINER_CLASS = 'dx-chat-messagebox-textarea-container';
+
+export const TYPING_END_DELAY = 2000;
+const ESCAPE_KEY = 'escape';
+
 class MessageBox extends DOMComponent<MessageBox, Properties> {
   _textArea!: ChatTextArea;
 
@@ -73,6 +78,8 @@ class MessageBox extends DOMComponent<MessageBox, Properties> {
       hoverStateEnabled: true,
       fileUploaderOptions: undefined,
       previewText: '',
+      speechToTextEnabled: false,
+      speechToTextOptions: undefined,
       text: '',
       onMessageEntered: undefined,
       onMessageEditCanceled: undefined,
@@ -163,6 +170,8 @@ class MessageBox extends DOMComponent<MessageBox, Properties> {
       focusStateEnabled,
       hoverStateEnabled,
       previewText,
+      speechToTextEnabled,
+      speechToTextOptions,
       text,
     } = this.option();
 
@@ -173,6 +182,8 @@ class MessageBox extends DOMComponent<MessageBox, Properties> {
       hoverStateEnabled,
       // eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing
       value: previewText || text,
+      speechToTextEnabled,
+      speechToTextOptions,
       onInput: (e: InputEvent): void => {
         this._triggerTypingStartAction(e);
         this._updateTypingEndTimeout();
@@ -287,6 +298,11 @@ class MessageBox extends DOMComponent<MessageBox, Properties> {
         this._createTypingEndAction();
         break;
 
+      case 'speechToTextEnabled':
+      case 'speechToTextOptions':
+        this._textArea.option(fullName, value);
+        break;
+
       case 'previewText':
         this._textArea.option('value', value);
         this._updateEditingPreview(value);