Detect UI elements

Detect and identify interactive UI elements in the current screen. Note: This feature currently only supports element detection within a running browser. If the browser is not running, the Elements array will be empty.

POST /boxes/{boxId}/actions/elements/detect

JavaScript

JavaScript

import GboxSDK from "gbox-sdk";

const gboxSDK = new GboxSDK({
  apiKey: process.env["GBOX_API_KEY"] // This is the default and can be omitted
});

async function main() {
  const box = await gboxSDK.create({ type: "linux" });

  await box.browser.openTab({
    url: "https://gbox.ai",
  });

  const { screenshot, elements } = await box.action.elements.detect({
    screenshot: {
      outputFormat: 'storageKey'
    }
  });

  console.info(`Screenshot: ${JSON.stringify(screenshot, null, 2)}`);

  console.info(`Detected elements length: ${elements.list().length}`);

  // You can send the screenshot to an LLM or Agent to decide which element to click
  const firstElement = elements.get("1");
  await box.action.click({
    // here we just click the first element
    target: firstElement,
  });

  console.info(
    `Clicked element: ${JSON.stringify(firstElement, null, 2)}`
  );
}

main();

import os
import json
from gbox_sdk import GboxSDK


def main():
    gbox_sdk = GboxSDK(api_key=os.environ["GBOX_API_KEY"])  # This is the default and can be omitted

    box = gbox_sdk.create(type="linux")

    new_tab = box.browser.open_tab(url="https://gbox.ai")

    res = box.action.elements.detect(
        screenshot={
            "output_format": "storageKey"
        }
    )

    print(f"Screenshot: {json.dumps(res.screenshot, indent=2)}")

    print(f"Detected elements length: {len(res.elements.list())}")

    # You can send the screenshot to an LLM or Agent to decide which element to click
    first_element = res.elements.get("1")
    box.action.click(target=first_element)

    print(f"Clicked element: {json.dumps(first_element, indent=2)}")


if __name__ == "__main__":
    main()

curl --request POST \
  --url https://gbox.ai/api/v1/boxes/{boxId}/actions/elements/detect \
  --header 'Authorization: Bearer <token>' \
  --header 'Content-Type: application/json' \
  --data '
{
  "screenshot": {
    "outputFormat": "base64",
    "presignedExpiresIn": "30m"
  }
}
'

200

{
  "screenshot": {
    "source": {
      "uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA...",
      "presignedUrl": "https://example.com/xxxxx/xxxxx/xxxxx"
    },
    "marked": {
      "uri": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAUA...",
      "presignedUrl": "https://example.com/xxxxx/xxxxx/xxxxx"
    }
  },
  "elements": [
    {
      "id": "1",
      "source": "chromium",
      "type": "button",
      "path": "#root > table > tbody > tr:nth-child(1) > td:nth-child(1) > div > button",
      "width": 100,
      "height": 50,
      "x": 100,
      "y": 100,
      "centerX": 150,
      "centerY": 125,
      "label": "Click me"
    }
  ]
}

Authorizations

Authorization · string · header · required

Enter your API Key in the format: Bearer . Get it from https://gbox.ai

Path Parameters

boxId · string · required

Box ID

Example:

"c9bdc193-b54b-4ddb-a035-5ac0c598d32d"

Body

application/json

Detect UI elements action configuration

screenshot · Detect Elements Screenshot Options · object

Screenshot options

Show child attributes

Example:

{
  "outputFormat": "base64",
  "presignedExpiresIn": "30m"
}

Response

200 - application/json

Result containing original screenshot, annotated screenshot, and detected elements

screenshot · Detected Elements Screenshot · object · required

Detected elements screenshot

Show child attributes

elements · Detected Element · object[] · required

Detected UI elements

Show child attributes