send keepalive when pressing the key

hidrpc.go

@@ -29,6 +29,8 @@ func handleHidRPCMessage(message hidrpc.Message, session *Session) {
 			session.reportHidRPCKeysDownState(*keysDownState)
 		}
 		rpcErr = err
+	case hidrpc.TypeKeypressKeepAliveReport:
+		gadget.DelayAutoRelease()
 	case hidrpc.TypePointerReport:
 		pointerReport, err := message.PointerReport()
 		if err != nil {

internal/hidrpc/hidrpc.go

@@ -10,14 +10,15 @@ import (
 type MessageType byte
 
 const (
-	TypeHandshake        MessageType = 0x01
+	TypeHandshake               MessageType = 0x01
-	TypeKeyboardReport   MessageType = 0x02
+	TypeKeyboardReport          MessageType = 0x02
-	TypePointerReport    MessageType = 0x03
+	TypePointerReport           MessageType = 0x03
-	TypeWheelReport      MessageType = 0x04
+	TypeWheelReport             MessageType = 0x04
-	TypeKeypressReport   MessageType = 0x05
+	TypeKeypressReport          MessageType = 0x05
-	TypeMouseReport      MessageType = 0x06
+	TypeKeypressKeepAliveReport MessageType = 0x09
-	TypeKeyboardLedState MessageType = 0x32
+	TypeMouseReport             MessageType = 0x06
-	TypeKeydownState     MessageType = 0x33
+	TypeKeyboardLedState        MessageType = 0x32
+	TypeKeydownState            MessageType = 0x33
 )
 
 const (
@@ -98,3 +99,11 @@ func NewKeydownStateMessage(state usbgadget.KeysDownState) *Message {
 		d: data,
 	}
 }
+
+// NewKeypressKeepAliveMessage creates a new keypress keep alive message.
+func NewKeypressKeepAliveMessage() *Message {
+	return &Message{
+		t: TypeKeypressKeepAliveReport,
+		d: []byte{},
+	}
+}

internal/usbgadget/hid_keyboard.go

@@ -173,33 +173,47 @@ func (u *UsbGadget) SetOnKeysDownChange(f func(state KeysDownState)) {
 	u.onKeysDownChange = &f
 }
 
-const autoReleaseKeyboardInterval = time.Millisecond * 300
+const autoReleaseKeyboardInterval = time.Millisecond * 450
 
 func (u *UsbGadget) scheduleAutoRelease(key byte) {
-	u.keysAutoReleaseLock.Lock()
+	u.kbdAutoReleaseLock.Lock()
-	defer u.keysAutoReleaseLock.Unlock()
+	defer u.kbdAutoReleaseLock.Unlock()
 
-	if u.keysAutoReleaseTimer != nil {
+	if u.kbdAutoReleaseTimer != nil {
-		u.keysAutoReleaseTimer.Stop()
+		u.kbdAutoReleaseTimer.Stop()
 	}
 
-	u.keysAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() {
+	u.kbdAutoReleaseTimer = time.AfterFunc(autoReleaseKeyboardInterval, func() {
 		u.performAutoRelease(key)
 	})
 }
 
 func (u *UsbGadget) cancelAutoRelease() {
-	u.keysAutoReleaseLock.Lock()
+	u.kbdAutoReleaseLock.Lock()
-	defer u.keysAutoReleaseLock.Unlock()
+	defer u.kbdAutoReleaseLock.Unlock()
 
-	if u.keysAutoReleaseTimer != nil {
+	if u.kbdAutoReleaseTimer != nil {
-		u.keysAutoReleaseTimer.Stop()
+		u.kbdAutoReleaseTimer.Stop()
 	}
 }
 
+func (u *UsbGadget) DelayAutoRelease() {
+	u.kbdAutoReleaseLock.Lock()
+	defer u.kbdAutoReleaseLock.Unlock()
+
+	u.log.Info().Msg("delaying auto-release")
+
+	if u.kbdAutoReleaseTimer == nil {
+		return
+	}
+
+	u.log.Info().Msg("resetting auto-release timer")
+	u.kbdAutoReleaseTimer.Reset(autoReleaseKeyboardInterval)
+}
+
 func (u *UsbGadget) performAutoRelease(key byte) {
-	u.keysAutoReleaseLock.Lock()
+	u.kbdAutoReleaseLock.Lock()
-	defer u.keysAutoReleaseLock.Unlock()
+	defer u.kbdAutoReleaseLock.Unlock()
 
 	select {
 	case <-u.keyboardStateCtx.Done():
@@ -207,12 +221,12 @@ func (u *UsbGadget) performAutoRelease(key byte) {
 	default:
 	}
 
-	_, err := u.KeypressReport(key, false)
+	_, err := u.keypressReport(key, false, false)
 	if err != nil {
 		u.log.Warn().Uint8("key", key).Msg("failed to auto-release keyboard key")
 	}
 
-	u.keysAutoReleaseTimer = nil
+	u.kbdAutoReleaseTimer = nil
 
 	u.log.Trace().Uint8("key", key).Msg("auto release performed")
 }
@@ -375,11 +389,21 @@ var KeyCodeToMaskMap = map[byte]byte{
 	RightSuper:   ModifierMaskRightSuper,
 }
 
-func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) {
+func (u *UsbGadget) keypressReport(key byte, press bool, autoRelease bool) (KeysDownState, error) {
+	ll := u.log.Info().Str("component", "kbd")
+	ll.Uint8("key", key).Msg("locking keyboardLock")
+
 	u.keyboardLock.Lock()
-	defer u.keyboardLock.Unlock()
+	defer func() {
+		u.keyboardLock.Unlock()
+		ll.Uint8("key", key).Msg("unlocked keyboardLock")
+	}()
+
+	ll.Uint8("key", key).Msg("resetting user input time")
 	defer u.resetUserInputTime()
 
+	ll.Uint8("key", key).Msg("locked keyboardLock")
+
 	// IMPORTANT: This code parallels the logic in the kernel's hid-gadget driver
 	// for handling key presses and releases. It ensures that the USB gadget
 	// behaves similarly to a real USB HID keyboard. This logic is paralleled
@@ -437,16 +461,54 @@ func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error)
 		}
 	}
 
+	ll.Uint8("key", key).Msg("checking if auto-release is enabled")
+
+	// if autoRelease {
+	// 	u.kbdAutoReleaseLock.Lock()
+	// 	u.kbdAutoReleaseLock.Unlock()
+	// 	ll.Uint8("key", key).Msg("locking kbdAutoReleaseLock, autoReleasLastKey reset")
+
+	// 	defer func() {
+	// 		ll.Uint8("key", key).Msg("unlocked kbdAutoReleaseLock, autoReleasLastKey reset")
+	// 	}()
+
+	// 	if u.kbdAutoReleaseLastKey == key {
+	// 		ll.Uint8("key", key).Msg("key already released by auto-release, skipping")
+	// 		u.kbdAutoReleaseLastKey = 0
+
+	// 		return u.UpdateKeysDown(modifier, keys), nil
+	// 	}
+	// }
+
+	ll.Uint8("key", key).Msg("writing keypress report to hidg0")
+
 	err := u.keyboardWriteHidFile(modifier, keys)
 	if err != nil {
 		u.log.Warn().Uint8("modifier", modifier).Uints8("keys", keys).Msg("Could not write keypress report to hidg0")
 	}
 
 	if press {
-		u.scheduleAutoRelease(key)
+		{
+			u.kbdAutoReleaseLock.Lock()
+			u.kbdAutoReleaseLastKey = key
+			u.kbdAutoReleaseLock.Unlock()
+		}
+
+		if autoRelease {
+			ll.Uint8("key", key).Msg("scheduling auto-release")
+			u.scheduleAutoRelease(key)
+		}
 	} else {
-		u.cancelAutoRelease()
+		if autoRelease {
+			ll.Uint8("key", key).Msg("canceling auto-release")
+			u.cancelAutoRelease()
+			ll.Uint8("key", key).Msg("auto-release canceled")
+		}
 	}
 
 	return u.UpdateKeysDown(modifier, keys), err
 }
+
+func (u *UsbGadget) KeypressReport(key byte, press bool) (KeysDownState, error) {
+	return u.keypressReport(key, press, true)
+}

internal/usbgadget/usbgadget.go

@@ -68,8 +68,9 @@ type UsbGadget struct {
 	keyboardState byte          // keyboard latched state (NumLock, CapsLock, ScrollLock, Compose, Kana)
 	keysDownState KeysDownState // keyboard dynamic state (modifier keys and pressed keys)
 
-	keysAutoReleaseLock  sync.Mutex
+	kbdAutoReleaseLock    sync.Mutex
-	keysAutoReleaseTimer *time.Timer
+	kbdAutoReleaseTimer   *time.Timer
+	kbdAutoReleaseLastKey byte
 
 	keyboardStateLock   sync.Mutex
 	keyboardStateCtx    context.Context
@@ -161,12 +162,12 @@ func (u *UsbGadget) Close() error {
 	}
 
 	// Stop auto-release timer
-	u.keysAutoReleaseLock.Lock()
+	u.kbdAutoReleaseLock.Lock()
-	if u.keysAutoReleaseTimer != nil {
+	if u.kbdAutoReleaseTimer != nil {
-		u.keysAutoReleaseTimer.Stop()
+		u.kbdAutoReleaseTimer.Stop()
-		u.keysAutoReleaseTimer = nil
+		u.kbdAutoReleaseTimer = nil
 	}
-	u.keysAutoReleaseLock.Unlock()
+	u.kbdAutoReleaseLock.Unlock()
 
 	// Close HID files
 	if u.keyboardHidFile != nil {

ui/src/hooks/hidRpc.ts

@@ -6,6 +6,7 @@ export const HID_RPC_MESSAGE_TYPES = {
     PointerReport: 0x03,
     WheelReport: 0x04,
     KeypressReport: 0x05,
+    KeypressKeepAliveReport: 0x09,
     MouseReport: 0x06,
     KeyboardLedState: 0x32,
     KeysDownState: 0x33,
@@ -278,12 +279,23 @@ export class MouseReportMessage extends RpcMessage {
     }
 }
 
+export class KeypressKeepAliveMessage extends RpcMessage {
+    constructor() {
+        super(HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport);
+    }
+
+    marshal(): Uint8Array {
+        return new Uint8Array([this.messageType]);
+    }
+}
+
 export const messageRegistry = {
     [HID_RPC_MESSAGE_TYPES.Handshake]: HandshakeMessage,
     [HID_RPC_MESSAGE_TYPES.KeysDownState]: KeysDownStateMessage,
     [HID_RPC_MESSAGE_TYPES.KeyboardLedState]: KeyboardLedStateMessage,
     [HID_RPC_MESSAGE_TYPES.KeyboardReport]: KeyboardReportMessage,
     [HID_RPC_MESSAGE_TYPES.KeypressReport]: KeypressReportMessage,
+    [HID_RPC_MESSAGE_TYPES.KeypressKeepAliveReport]: KeypressKeepAliveMessage,
 }
 
 export const unmarshalHidRpcMessage = (data: Uint8Array): RpcMessage | undefined => {

ui/src/hooks/useHidRpc.ts

@@ -6,6 +6,7 @@ import {
   HID_RPC_VERSION,
   HandshakeMessage,
   KeyboardReportMessage,
+  KeypressKeepAliveMessage,
   KeypressReportMessage,
   MouseReportMessage,
   PointerReportMessage,
@@ -13,6 +14,8 @@ import {
   unmarshalHidRpcMessage,
 } from "./hidRpc";
 
+const KEEPALIVE_MESSAGE = new KeypressKeepAliveMessage();
+
 export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
   const { rpcHidChannel, setRpcHidProtocolVersion, rpcHidProtocolVersion } = useRTCStore();
   const rpcHidReady = useMemo(() => {
@@ -68,6 +71,10 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
     [sendMessage],
   );
 
+  const reportKeypressKeepAlive = useCallback(() => {
+    sendMessage(KEEPALIVE_MESSAGE);
+  }, [sendMessage]);
+
   const sendHandshake = useCallback(() => {
     if (rpcHidProtocolVersion) return;
     if (!rpcHidChannel) return;
@@ -143,6 +150,7 @@ export function useHidRpc(onHidRpcMessage?: (payload: RpcMessage) => void) {
     reportKeypressEvent,
     reportAbsMouseEvent,
     reportRelMouseEvent,
+    reportKeypressKeepAlive,
     rpcHidProtocolVersion,
     rpcHidReady,
     rpcHidStatus,

ui/src/hooks/useKeyboard.ts

@@ -1,4 +1,4 @@
-import { useCallback } from "react";
+import { useCallback, useRef } from "react";
 
 import { hidErrorRollOver, hidKeyBufferSize, KeysDownState, useHidStore, useRTCStore } from "@/hooks/stores";
 import { JsonRpcResponse, useJsonRpc } from "@/hooks/useJsonRpc";
@@ -11,6 +11,9 @@ export default function useKeyboard() {
   const { rpcDataChannel } = useRTCStore();
   const { keysDownState, setKeysDownState, setKeyboardLedState } = useHidStore();
 
+  // Keepalive timer management
+  const keepAliveTimerRef = useRef<number | null>(null);
+
   // INTRODUCTION: The earlier version of the JetKVM device shipped with all keyboard state
   // being tracked on the browser/client-side. When adding the keyPressReport API to the
   // device-side code, we have to still support the situation where the browser/client-side code
@@ -26,6 +29,7 @@ export default function useKeyboard() {
   const {
     reportKeyboardEvent: sendKeyboardEventHidRpc,
     reportKeypressEvent: sendKeypressEventHidRpc,
+    reportKeypressKeepAlive: sendKeypressKeepAliveHidRpc,
     rpcHidReady,
   } = useHidRpc((message) => {
     switch (message.constructor) {
@@ -70,17 +74,6 @@ export default function useKeyboard() {
     ],
   );
 
-  // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
-  // This is useful for macros and when the browser loses focus to ensure that the keyboard state
-  // is clean.
-  const resetKeyboardState = useCallback(
-    async () => {
-      // Reset the keys buffer to zeros and the modifier state to zero
-      keysDownState.keys.length = hidKeyBufferSize;
-      keysDownState.keys.fill(0);
-      keysDownState.modifier = 0;
-      sendKeyboardEvent(keysDownState);
-    }, [keysDownState, sendKeyboardEvent]);
 
   // executeMacro is used to execute a macro consisting of multiple steps.
   // Each step can have multiple keys, multiple modifiers and a delay.
@@ -111,12 +104,61 @@ export default function useKeyboard() {
     }
   };
 
+  const KEEPALIVE_INTERVAL = 200; // 200ms interval
+
+  const cancelKeepAlive = useCallback(() => {
+    if (keepAliveTimerRef.current) {
+      clearInterval(keepAliveTimerRef.current);
+      keepAliveTimerRef.current = null;
+    }
+  }, []);
+
+  const scheduleKeepAlive = useCallback(() => {
+    // Clear existing timer if it exists
+    if (keepAliveTimerRef.current) {
+      clearInterval(keepAliveTimerRef.current);
+    }
+
+    // Create new interval timer
+    keepAliveTimerRef.current = setInterval(() => {
+      sendKeypressKeepAliveHidRpc();
+    }, KEEPALIVE_INTERVAL);
+  }, [sendKeypressKeepAliveHidRpc]);
+
+  // resetKeyboardState is used to reset the keyboard state to no keys pressed and no modifiers.
+  // This is useful for macros and when the browser loses focus to ensure that the keyboard state
+  // is clean.
+  const resetKeyboardState = useCallback(async () => {
+    // Cancel keepalive since we're resetting the keyboard state
+    cancelKeepAlive();
+
+    // Reset the keys buffer to zeros and the modifier state to zero
+    keysDownState.keys.length = hidKeyBufferSize;
+    keysDownState.keys.fill(0);
+    keysDownState.modifier = 0;
+    sendKeyboardEvent(keysDownState);
+  }, [keysDownState, sendKeyboardEvent, cancelKeepAlive]);
+
   // handleKeyPress is used to handle a key press or release event.
   // This function handle both key press and key release events.
   // It checks if the keyPressReport API is available and sends the key press event.
   // If the keyPressReport API is not available, it simulates the device-side key
   // handling for legacy devices and updates the keysDownState accordingly.
   // It then sends the full keyboard state to the device.
+
+  const sendKeypress = useCallback(
+    (key: number, press: boolean) => {
+      cancelKeepAlive();
+
+      sendKeypressEventHidRpc(key, press);
+
+      if (press) {
+        scheduleKeepAlive();
+      }
+    },
+    [sendKeypressEventHidRpc, scheduleKeepAlive, cancelKeepAlive],
+  );
+
   const handleKeyPress = useCallback(
     async (key: number, press: boolean) => {
       if (rpcDataChannel?.readyState !== "open" && !rpcHidReady) return;
@@ -129,7 +171,7 @@ export default function useKeyboard() {
         // Older device version doesn't support this API, so we will switch to local key handling
         // In that case we will switch to local key handling and update the keysDownState
         // in client/browser-side code using simulateDeviceSideKeyHandlingForLegacyDevices.
-        sendKeypressEventHidRpc(key, press);
+        sendKeypress(key, press);
       } else {
         // if the keyPress api is not available, we need to handle the key locally
         const downState = simulateDeviceSideKeyHandlingForLegacyDevices(keysDownState, key, press);
@@ -147,7 +189,7 @@ export default function useKeyboard() {
       resetKeyboardState,
       rpcDataChannel?.readyState,
       sendKeyboardEvent,
-      sendKeypressEventHidRpc,
+      sendKeypress,
     ],
   );
 
@@ -210,5 +252,10 @@ export default function useKeyboard() {
     return { modifier: modifiers, keys };
   }
 
-  return { handleKeyPress, resetKeyboardState, executeMacro };
+  // Cleanup function to cancel keepalive timer
+  const cleanup = useCallback(() => {
+    cancelKeepAlive();
+  }, [cancelKeepAlive]);
+
+  return { handleKeyPress, resetKeyboardState, executeMacro, cleanup };
 }