Merge pull request #214 from Ayush0Chaudhary/minor-fixes

Ayush0Chaudhary · web-flow · commit 4f0d755df299 · 2025-09-11T05:45:18.000+05:30
⏏️ Implemented the long press element, and the fixed the scroll AAAHH!
diff --git a/app/src/main/java/com/blurr/voice/ScreenInteractionService.kt b/app/src/main/java/com/blurr/voice/ScreenInteractionService.kt
@@ -562,6 +562,30 @@ class ScreenInteractionService : AccessibilityService() {
 
         dispatchGesture(gesture, null, null)
     }
+    /**
+     * Performs a long press gesture at a specific point on the screen.
+     * @param x The x-coordinate of the long press.
+     * @param y The y-coordinate of the long press.
+     */
+    fun longClickOnPoint(x: Float, y: Float) {
+        // Show visual feedback for the tap if the debug flag is enabled
+        if (DEBUG_SHOW_TAPS) {
+            showDebugTap(x, y)
+        }
+
+        val path = Path().apply {
+            moveTo(x, y)
+        }
+        // A long press is essentially a tap that is held down.
+        // 600ms is a common duration for a long press.
+        val longPressStroke = GestureDescription.StrokeDescription(path, 0, 2000L)
+
+        val gesture = GestureDescription.Builder()
+            .addStroke(longPressStroke)
+            .build()
+
+        dispatchGesture(gesture, null, null)
+    }
 
     /**
      * Scrolls the screen down by a given number of pixels with more precision.
diff --git a/app/src/main/java/com/blurr/voice/api/Finger.kt b/app/src/main/java/com/blurr/voice/api/Finger.kt
@@ -93,6 +93,15 @@ class Finger(private val context: Context) {
         service?.clickOnPoint(x.toFloat(), y.toFloat())
     }
 
+    /**
+     * Performs a long press (press and hold) at a specific point on the screen.
+     */
+    fun longPress(x: Int, y: Int) {
+        Log.d(TAG, "Long pressing at ($x, $y)")
+        // This assumes your ScreenInteractionService has a method `longClickOnPoint`
+        service?.longClickOnPoint(x.toFloat(), y.toFloat())
+    }
+
     /**
      * Swipes between two points on the screen.
      */
@@ -150,7 +159,7 @@ class Finger(private val context: Context) {
      * @param pixels The number of pixels to scroll.
      * @param duration The duration of the swipe in milliseconds.
      */
-    fun scrollDown(pixels: Int, duration: Int = 500) {
+    fun scrollUp(pixels: Int, duration: Int = 500) {
         val displayMetrics = context.resources.displayMetrics
         val screenWidth = displayMetrics.widthPixels
         val screenHeight = displayMetrics.heightPixels
@@ -173,7 +182,7 @@ class Finger(private val context: Context) {
      * @param pixels The number of pixels to scroll.
      * @param duration The duration of the swipe in milliseconds.
      */
-    fun scrollUp(pixels: Int, duration: Int = 500) {
+    fun scrollDown(pixels: Int, duration: Int = 500) {
         val displayMetrics = context.resources.displayMetrics
         val screenWidth = displayMetrics.widthPixels
         val screenHeight = displayMetrics.heightPixels
diff --git a/app/src/main/java/com/blurr/voice/utilities/STTManager.kt b/app/src/main/java/com/blurr/voice/utilities/STTManager.kt
@@ -22,10 +22,12 @@ class STTManager(private val context: Context) {
     private var onPartialResultCallback: ((String) -> Unit)? = null
     private var isInitialized = false
     private val visualizerManager = STTVisualizer(context)
+    private val COMPLETE_SILENCE_MS = 2500  // time of silence to consider input complete
+    private val POSSIBLE_SILENCE_MS = 2000  // shorter silence hint window
+    private val MIN_UTTERANCE_MS     = 1500 // enforce a minimum listening duration
 
     
-    // Remove initialization from constructor - will be done lazily on main thread
-    
+
     private fun initializeSpeechRecognizer() {
         if (isInitialized) return
         
@@ -58,12 +60,10 @@ class STTManager(private val context: Context) {
             }
 
             override fun onRmsChanged(rmsdB: Float) {
-                // --- NEW: Invoke the callback with the new audio level ---
                 visualizerManager.onRmsChanged(rmsdB)
             }
             
             override fun onBufferReceived(buffer: ByteArray?) {
-                // Optional: Can be used for real-time processing
             }
             
             override fun onEndOfSpeech() {
@@ -113,7 +113,6 @@ class STTManager(private val context: Context) {
             }
 
             override fun onPartialResults(partialResults: Bundle?) {
-                // v-- IMPLEMENT THIS METHOD --v
                 val matches = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
                 if (!matches.isNullOrEmpty()) {
                     val partialText = matches[0]
@@ -123,7 +122,6 @@ class STTManager(private val context: Context) {
             }
             
             override fun onEvent(eventType: Int, params: Bundle?) {
-                // Optional: Handle specific events
             }
         }
     }
@@ -144,7 +142,6 @@ class STTManager(private val context: Context) {
         this.onListeningStateChange = onListeningStateChange
         this.onPartialResultCallback = onPartialResult
 
-        // Initialize on main thread if needed
         CoroutineScope(Dispatchers.Main).launch {
             initializeSpeechRecognizer()
             
@@ -160,6 +157,9 @@ class STTManager(private val context: Context) {
                 putExtra(RecognizerIntent.EXTRA_LANGUAGE, Locale.getDefault())
                 putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)
                 putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1)
+//                putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, COMPLETE_SILENCE_MS)
+//                putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, POSSIBLE_SILENCE_MS)
+//                putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_MINIMUM_LENGTH_MILLIS, MIN_UTTERANCE_MS)
             }
             
             try {
diff --git a/app/src/main/java/com/blurr/voice/v2/Agent.kt b/app/src/main/java/com/blurr/voice/v2/Agent.kt
@@ -10,6 +10,7 @@ import com.blurr.voice.v2.llm.GeminiApi
 import com.blurr.voice.v2.llm.GeminiMessage
 import com.blurr.voice.v2.message_manager.MemoryManager
 import com.blurr.voice.v2.perception.Perception
+import com.blurr.voice.utilities.SpeechCoordinator
 import kotlinx.coroutines.delay
 
 /**
@@ -37,6 +38,9 @@ class Agent(
     // The agent's internal state, which is updated at each step.
     val state: AgentState = AgentState()
     private val TAG = "AgentV2"
+    
+    // Speech coordinator for voice notifications
+    private val speechCoordinator = SpeechCoordinator.getInstance(context)
 
     // A complete, long-term record of the entire session.
     // We use <Unit> because we haven't defined a custom structured output for the 'done' action yet.
@@ -83,6 +87,7 @@ class Agent(
                 memoryManager.addContextMessage(GeminiMessage(text = "System Note: Your previous output was not valid JSON. Please ensure your response is correctly formatted."))
                 if (state.consecutiveFailures >= settings.maxFailures) {
                     Log.d(TAG,"❌ Agent failed too many times consecutively. Stopping.")
+                    speechCoordinator.speakToUser("Agent failed after multiple attempts. Stopping execution.")
                     break
                 }
                 delay(1000) // Wait a moment before retrying
@@ -122,6 +127,7 @@ class Agent(
             // --- Check for Task Completion ---
             if (actionResults.any { it.isDone == true }) {
                 Log.d(TAG,"✅ Agent finished the task.")
+                speechCoordinator.speakToUser("Task completed successfully.")
                 state.stopped = true
             }
 
@@ -131,8 +137,8 @@ class Agent(
 
         // --- Loop Finished ---
         if (state.nSteps > maxSteps) {
-
             Log.d(TAG,"--- 🏁 Agent reached max steps. Stopping. ---")
+            speechCoordinator.speakToUser("Agent reached maximum steps limit. Stopping execution.")
         } else {
             Log.d(TAG,"--- 🏁 Agent run finished. ---")
         }
diff --git a/app/src/main/java/com/blurr/voice/v2/AgentService.kt b/app/src/main/java/com/blurr/voice/v2/AgentService.kt
@@ -3,6 +3,7 @@ package com.blurr.voice.v2
 import android.app.Notification
 import android.app.NotificationChannel
 import android.app.NotificationManager
+import android.app.PendingIntent
 import android.app.Service
 import android.content.Context
 import android.content.Intent
@@ -84,8 +85,8 @@ class AgentService : Service() {
 //
 //
     companion object {
-        private const val NOTIFICATION_CHANNEL_ID = "AgentServiceChannel"
-        private const val NOTIFICATION_ID = 1
+        private const val NOTIFICATION_CHANNEL_ID = "AgentServiceChannelV2"
+        private const val NOTIFICATION_ID = 14
         private const val EXTRA_TASK = "com.blurr.voice.v2.EXTRA_TASK"
         private const val ACTION_STOP_SERVICE = "com.blurr.voice.v2.ACTION_STOP_SERVICE"
 
@@ -243,9 +244,26 @@ class AgentService : Service() {
      * Creates the persistent notification for the foreground service.
      */
     private fun createNotification(contentText: String): Notification {
+        // Create PendingIntent for the stop action
+        val stopIntent = Intent(this, AgentService::class.java).apply {
+            action = ACTION_STOP_SERVICE
+        }
+        val stopPendingIntent = PendingIntent.getService(
+            this,
+            0,
+            stopIntent,
+            PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE
+        )
+
         return NotificationCompat.Builder(this, NOTIFICATION_CHANNEL_ID)
             .setContentTitle("AI Agent Active")
             .setContentText(contentText)
+            .addAction(
+                android.R.drawable.ic_media_pause, // Using built-in pause icon as stop button
+                "Stop Agent",
+                stopPendingIntent
+            )
+            .setOngoing(true) // Makes notification persistent and harder to dismiss
             // .setSmallIcon(R.drawable.ic_agent_notification) // TODO: Add a notification icon
             .build()
     }
diff --git a/app/src/main/java/com/blurr/voice/v2/actions/Action.kt b/app/src/main/java/com/blurr/voice/v2/actions/Action.kt
@@ -22,6 +22,7 @@ data class ParamSpec(val name: String, val type: KClass<*>, val description: Str
 sealed class Action {
     // Each action is a data class (if it has args) or an object (if it doesn't).
     // Note: Property names here follow Kotlin's camelCase convention.
+    data class LongPressElement(val elementId: Int) : Action()
     data class TapElement(val elementId: Int) : Action()
     data object SwitchApp : Action()
     data object Back : Action()
@@ -129,16 +130,22 @@ sealed class Action {
                 params = listOf(ParamSpec("app_name", String::class, "The name of the app.")),
                 build = { args -> OpenApp(args["app_name"] as String) }
             ),
-            "scroll_down" to Spec(
-                name = "scroll_down",
-                description = "Scroll down by the specified amount of pixels.",
-                params = listOf(ParamSpec("amount", Int::class, "Amount of pixels to scroll down.")),
+            "swipe_down" to Spec(
+                name = "swipe_down",
+                description = "swipe down by the specified amount of pixels.",
+                params = listOf(ParamSpec("amount", Int::class, "Amount of pixels to swipe down.")),
                 build = { args -> ScrollDown(args["amount"] as Int) }
             ),
-            "scroll_up" to Spec(
-                name = "scroll_up",
-                description = "Scroll up by the specified amount of pixels.",
-                params = listOf(ParamSpec("amount", Int::class, "Amount of pixels to scroll up.")),
+            "long_press_element" to Spec(
+                name = "long_press_element",
+                description = "Press and hold the element with the specified numeric ID. Useful for context menus, selecting text, etc.",
+                params = listOf(ParamSpec("element_id", Int::class, "The numeric ID of the element to long press.")),
+                build = { args -> LongPressElement(args["element_id"] as Int) }
+            ),
+            "swipe_up" to Spec(
+                name = "swipe_up",
+                description = "swipe up by the specified amount of pixels.",
+                params = listOf(ParamSpec("amount", Int::class, "Amount of pixels to swipe up.")),
                 build = { args -> ScrollUp(args["amount"] as Int) }
             ),
             "search_google" to Spec(
diff --git a/app/src/main/java/com/blurr/voice/v2/actions/ActionExecutor.kt b/app/src/main/java/com/blurr/voice/v2/actions/ActionExecutor.kt
@@ -117,6 +117,27 @@ class ActionExecutor(private val finger: Finger) {
                     includeExtractedContentOnlyOnce = true
                 )
             }
+            is Action.LongPressElement -> {
+                val elementNode = screenAnalysis.elementMap[action.elementId]
+                if (elementNode != null) {
+                    val bounds = elementNode.attributes["bounds"]
+                    val text = elementNode.getVisibleText().replace("\n", " ")
+                    val resourceId = elementNode.attributes["resource-id"] ?: ""
+                    val extraInfo = elementNode.extraInfo
+                    val className = (elementNode.attributes["class"] ?: "").removePrefix("android.")
+
+                    if (bounds != null) {
+                        val (centerX, centerY) = getCenterFromBounds(bounds)
+                        // Assuming finger has a longPress method. Adjust if necessary.
+                        finger.longPress(centerX, centerY)
+                        ActionResult(longTermMemory = "Long-pressed element text:$text <$resourceId> <$extraInfo> <$className>")
+                    } else {
+                        ActionResult(error = "Element with ID ${action.elementId} has no bounds information.")
+                    }
+                } else {
+                    ActionResult(error = "Element with ID ${action.elementId} not found in the current screen state.")
+                }
+            }
             is Action.OpenApp -> {
                 val packageName = findPackageNameFromAppName(action.appName, context)
                 if (packageName != null) {

Original file line number	Diff line number	Diff line change
`@@ -22,10 +22,12 @@ class STTManager(private val context: Context) {`
`22`	`22`	`private var onPartialResultCallback: ((String) -> Unit)? = null`
`23`	`23`	`private var isInitialized = false`
`24`	`24`	`private val visualizerManager = STTVisualizer(context)`
	`25`	`+ private val COMPLETE_SILENCE_MS = 2500 // time of silence to consider input complete`
	`26`	`+ private val POSSIBLE_SILENCE_MS = 2000 // shorter silence hint window`
	`27`	`+ private val MIN_UTTERANCE_MS = 1500 // enforce a minimum listening duration`
`25`	`28`
`26`	`29`
`27`		`- // Remove initialization from constructor - will be done lazily on main thread`
`28`		`-`
	`30`	`+`
`29`	`31`	`private fun initializeSpeechRecognizer() {`
`30`	`32`	`if (isInitialized) return`
`31`	`33`
`@@ -58,12 +60,10 @@ class STTManager(private val context: Context) {`
`58`	`60`	`}`
`59`	`61`
`60`	`62`	`override fun onRmsChanged(rmsdB: Float) {`
`61`		`- // --- NEW: Invoke the callback with the new audio level ---`
`62`	`63`	`visualizerManager.onRmsChanged(rmsdB)`
`63`	`64`	`}`
`64`	`65`
`65`	`66`	`override fun onBufferReceived(buffer: ByteArray?) {`
`66`		`- // Optional: Can be used for real-time processing`
`67`	`67`	`}`
`68`	`68`
`69`	`69`	`override fun onEndOfSpeech() {`
`@@ -113,7 +113,6 @@ class STTManager(private val context: Context) {`
`113`	`113`	`}`
`114`	`114`
`115`	`115`	`override fun onPartialResults(partialResults: Bundle?) {`
`116`		`- // v-- IMPLEMENT THIS METHOD --v`
`117`	`116`	`val matches = partialResults?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)`
`118`	`117`	`if (!matches.isNullOrEmpty()) {`
`119`	`118`	`val partialText = matches[0]`
`@@ -123,7 +122,6 @@ class STTManager(private val context: Context) {`
`123`	`122`	`}`
`124`	`123`
`125`	`124`	`override fun onEvent(eventType: Int, params: Bundle?) {`
`126`		`- // Optional: Handle specific events`
`127`	`125`	`}`
`128`	`126`	`}`
`129`	`127`	`}`
`@@ -144,7 +142,6 @@ class STTManager(private val context: Context) {`
`144`	`142`	`this.onListeningStateChange = onListeningStateChange`
`145`	`143`	`this.onPartialResultCallback = onPartialResult`
`146`	`144`
`147`		`- // Initialize on main thread if needed`
`148`	`145`	`CoroutineScope(Dispatchers.Main).launch {`
`149`	`146`	`initializeSpeechRecognizer()`
`150`	`147`
`@@ -160,6 +157,9 @@ class STTManager(private val context: Context) {`
`160`	`157`	`putExtra(RecognizerIntent.EXTRA_LANGUAGE, Locale.getDefault())`
`161`	`158`	`putExtra(RecognizerIntent.EXTRA_PARTIAL_RESULTS, true)`
`162`	`159`	`putExtra(RecognizerIntent.EXTRA_MAX_RESULTS, 1)`
	`160`	`+// putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, COMPLETE_SILENCE_MS)`
	`161`	`+// putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, POSSIBLE_SILENCE_MS)`
	`162`	`+// putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_MINIMUM_LENGTH_MILLIS, MIN_UTTERANCE_MS)`
`163`	`163`	`}`
`164`	`164`
`165`	`165`	`try {`