CUA like agent with tool use and hint support. (#318)

* overlay_utils can return array if needed.

* exact goal loading in the tool-use-agent

* add tool use cua_like_agent

* remove unused imports

* make extra-dependency user facing and update dev dependency group

* update CI/CD env installation (dev is default group)

* update makefile to use uv

* black and remove unneeded items pip list from code formatting CI/CD.


---------

Co-authored-by: Patrice Bechard <[email protected]>

Author: amanjaiswal73892

.github/workflows/code_format.yml

@@ -24,13 +24,10 @@ jobs:
           enable-cache: true
 
       - name: Set up Python
-        run: uv python install 3.11
+        run: uv python install 3.12
 
       - name: Install dependencies
-        run: uv sync --frozen --extra dev
-
-      - name: List packages
-        run: uv pip list
+        run: uv sync --frozen 
 
       - name: Code Formatting
         run: uv run black src/ --check --diff

.github/workflows/darglint.yml

@@ -27,10 +27,7 @@ jobs:
         run: uv python install 3.12 # this fails in 3.11
 
       - name: Install dependencies
-        run: uv sync --frozen --extra dev
-
-      - name: List packages
-        run: uv pip list
+        run: uv sync --frozen 
 
       - name: Darglint checks
         run: uv run darglint -v 2 -z short src/

.github/workflows/unit_tests.yml

@@ -32,7 +32,7 @@ jobs:
         run: uv python install 3.11
 
       - name: Install AgentLab
-        run: uv sync --frozen --extra dev
+        run: uv sync --frozen 
 
       - name: List packages
         run: uv pip list

Makefile

@@ -1,14 +1,14 @@
 .PHONY: test setup miniwob lint stop-miniwob osworld
 
 setup:
-	@pip install -e .
-	@playwright install chromium --with-deps
-	@python -c 'import nltk; nltk.download("punkt_tab")'
+	@uv sync --python 3.12
+	@uv run playwright install chromium --with-deps
+	@uv run python -c 'import nltk; nltk.download("punkt_tab")'
 
 miniwob: stop-miniwob
 	@git clone https://github.com/Farama-Foundation/miniwob-plusplus.git || true
 	@cd miniwob-plusplus && git checkout 7fd85d71a4b60325c6585396ec4f48377d049838
-	@python -m http.server 8080 --directory miniwob-plusplus/miniwob/html & echo $$! > .miniwob-server.pid
+	@uv run python -m http.server 8080 --directory miniwob-plusplus/miniwob/html & echo $$! > .miniwob-server.pid
 	@sleep 3
 	@echo "MiniWob server started on http://localhost:8080"
 
@@ -22,14 +22,14 @@ stop-miniwob:
 	@echo "MiniWob server stopped"
 
 run-tests:
-	@MINIWOB_URL="http://localhost:8080/miniwob/" pytest -n 5 --durations=10 -m 'not pricy' tests/
+	@MINIWOB_URL="http://localhost:8080/miniwob/" uv run pytest -n 5 --durations=10 -m 'not pricy' tests/
 	@echo "Tests completed"
 
 test: setup miniwob check-miniwob run-tests stop-miniwob
 
 lint: setup
-	@black src/ --check --diff
-	@darglint -v 2 -z short src/
+	@uv run black src/ --check --diff
+	@uv run darglint -v 2 -z short src/
 
 osworld:
 	@echo "Setting up OSWorld..."
@@ -42,9 +42,9 @@ osworld:
 		sed -i.bak 's/tqdm~=.*/tqdm/' requirements.txt && \
 		sed -i.bak 's/pandas~=.*/pandas/' requirements.txt
 	@echo "Installing OSWorld requirements..."
-	@cd OSWorld && pip install -r requirements.txt
+	@cd OSWorld && uv pip install -r requirements.txt
 	@echo "Installing OSWorld in development mode..."
-	@cd OSWorld && pip install -e .
+	@cd OSWorld && uv pip install -e .
 	@echo "OSWorld setup completed!"
 	@echo "Next steps:"
 	@echo "1. Configure your VM (VMware/VirtualBox) according to OSWorld documentation"

pyproject.toml

@@ -49,24 +49,20 @@ dependencies = [
     "pillow",
     "gymnasium>=0.27",
     "torch>=2.2.2",
-    "safetensors>=0.4.0",
-    "transformers>=4.38.2",
     "anthropic>=0.62.0",
     "litellm>=1.75.3",
     "python-dotenv>=1.1.1",
 ]
 
 [project.optional-dependencies]
-dev = [
-    "black[jupyter]>=24.2.0",
-    "blacken-docs",
-    "pre-commit",
-    "pytest==7.3.2",
-    "flaky",
-    "pytest-xdist",
-    "pytest-playwright",
+hint = [
+    "sentence-transformers>=5.0.0",
+]
+transformers = [
+    "transformers>=4.38.2", 
 ]
 
+
 [project.urls]
 Homepage = "https://github.com/ServiceNow/AgentLab"
 
@@ -107,9 +103,13 @@ dev = [
     "darglint>=1.8.1",
     "ipykernel>=6.30.1",
     "pip>=25.2",
-]
-hint = [
-    "sentence-transformers>=5.0.0",
+    "black[jupyter]>=24.2.0",
+    "blacken-docs",
+    "pre-commit",
+    "pytest==7.3.2",
+    "flaky",
+    "pytest-xdist",
+    "pytest-playwright",
 ]
 
 

src/agentlab/agents/agent_utils.py

@@ -5,6 +5,7 @@
 
 from agentlab.analyze import overlay_utils
 from agentlab.llm.llm_utils import img_to_base_64
+import numpy as np
 
 
 def draw_mouse_pointer(image: Image.Image, x: int, y: int) -> Image.Image:
@@ -135,7 +136,7 @@ def zoom_webpage(page: Page, zoom_factor: float = 1.5):
     return page
 
 
-def overlay_action(obs, action):
+def overlay_action(obs, action, return_array=False):
     """Overlays actions on screenshot in-place"""
     act_img = copy.deepcopy(obs["screenshot"])
     act_img = Image.fromarray(act_img)
@@ -153,4 +154,7 @@ def overlay_action(obs, action):
                 pass
 
     overlay_utils.annotate_action(act_img, action, properties=new_obs_properties)
-    return img_to_base_64(act_img)
+    if return_array:
+        return np.array(act_img)
+    else:
+        return img_to_base_64(act_img)