Set-of-Mark

class OmniParser:
    def __init__(self, device: str = "auto"):
        """Initialize the parser with automatic device detection"""

    def parse(
        self,
        image: PIL.Image,
        box_threshold: float = 0.3,
        iou_threshold: float = 0.1,
        use_ocr: bool = True,
        ocr_engine: str = "easyocr"
    ) -> ParseResult:
        """Parse UI elements from an image"""

@dataclass
class ParseResult:
    elements: List[UIElement]      # Detected elements
    visualized_image: PIL.Image    # Annotated image
    processing_time: float         # Time in seconds

    def to_dict(self) -> dict:
        """Convert to JSON-serializable dictionary"""

    def filter_by_type(self, elem_type: str) -> List[UIElement]:
        """Filter elements by type ('icon' or 'text')"""

class UIElement(BaseModel):
    id: Optional[int] = Field(None)     # Element ID (1-indexed)
    type: Literal["icon", "text"]       # Element type
    bbox: BoundingBox                   # Bounding box coordinates { x1, y1, x2, y2 }
    interactivity: bool = Field(default=False)   # Whether the element is interactive
    confidence: float = Field(default=1.0)       # Detection confidence