Add a better rendering strategy for clickables

browser/browser.go

@@ -106,6 +106,7 @@ func (b *Browser) run(actions ...chromedp.Action) error {
 }
 
 func (b *Browser) Click(id virtualid.VirtualID) error {
+ previousLocation := b.display.Location
  if !b.vIDGenerator.IsValidVirtualID(id) {
  return fmt.Errorf("invalid virtual id: %s", id)
  } else if exists, err := b.DoesVirtualIDExist(string(id)); err != nil {
@@ -116,9 +117,18 @@ func (b *Browser) Click(id virtualid.VirtualID) error {
  return fmt.Errorf("error checking element type for virtual id: %w", err)
  } else if elementType != ElementTypeButton && elementType != ElementTypeLink {
  return fmt.Errorf("cannot click element type %s", elementType)
- } else {
- return b.ClickByVirtualID(string(id))
+ } else if err := b.ClickByVirtualID(string(id)); err != nil {
+ return fmt.Errorf("error clicking by virtual id: %w", err)
+ } else if err := b.updateDisplay(); err != nil {
+ return fmt.Errorf("error updating display: %w", err)
+ } else if previousLocation != b.display.Location {
+ if supportsAriaLabels, err := b.DoesSupportAriaLabels(); err != nil {
+ log.Println("error checking if browser supports aria labels:", err)
+ } else if !supportsAriaLabels {
+ log.Println("warning: browser does not support aria labels")
+ }
  }
+ return nil
 }
 
 func (b *Browser) SendKeys(id virtualid.VirtualID, keys string) error {
@@ -144,9 +154,14 @@ func (b *Browser) Navigate(URL string) error {
  return fmt.Errorf("error ensuring scheme: %w", err)
  } else if valid, err := IsValidURL(u); !valid {
  return fmt.Errorf("invalid url %s: %w", u, err)
- } else {
- return b.run(chromedp.Navigate(u), chromedp.Sleep(1*time.Second))
+ } else if err := b.run(chromedp.Navigate(u), chromedp.Sleep(1*time.Second)); err != nil {
+ return fmt.Errorf("error navigating to %s: %w", u, err)
+ } else if supportsAriaLabels, err := b.DoesSupportAriaLabels(); err != nil {
+ log.Println("error checking if browser supports aria labels:", err)
+ } else if !supportsAriaLabels {
+ log.Println("warning: browser does not support aria labels")
  }
+ return nil
 }
 
 func (b *Browser) addVirtualIDs() error {

browser/primitives.go

@@ -106,3 +106,18 @@ getAllVisibleVirtualIDs();`
  return virtualIDs, nil
  }
 }
+
+func (b *Browser) DoesSupportAriaLabels() (bool, error) {
+ js := `function doesSupportAriaLabels() {
+ const ariaLabelElems = document.querySelectorAll('[aria-label]');
+ return ariaLabelElems.length > 0;
+}
+doesSupportAriaLabels();
+`
+ var supportsAriaLabels bool
+ if err := b.run(chromedp.Evaluate(js, &supportsAriaLabels)); err != nil {
+ return false, fmt.Errorf("error checking if browser supports aria labels: %w", err)
+ } else {
+ return supportsAriaLabels, nil
+ }
+}

translators/html2md/html_to_md.go

@@ -61,21 +61,18 @@ func (t *HTML2MDTranslator) Visit(n *html.Node) string {
  case html.ElementNode:
  content := t.visitChildren(n)
  attrMap := buildAttrMapFromNode(n)
+ virtualID := attrMap["data-vid"]
  switch n.Data {
  case "button":
- innerText := parseInnerText(content)
- virtualID, ok := attrMap["data-vid"]
- if !ok {
+ if !isClickable(n, attrMap) {
  return strings.Join(content, "\n")
+ } else if label, isClickable := getLabelForClickable(n, attrMap, content); !isClickable {
+ return strings.Join(content, "\n")
+ } else {
+ return renderSelectable(SelectableTypeButton, virtualID, label, "")
  }
- if innerText == "" {
- return ""
- }
- return renderSelectable(SelectableTypeButton, virtualID, innerText, "")
  case "input", "textarea":
- if virtualID, ok := attrMap["data-vid"]; !ok {
- return strings.Join(content, "\n")
- } else if !isInputable(n, attrMap) {
+ if !isInputable(n, attrMap) {
  return strings.Join(content, "\n")
  } else if label, isInputable := getLabelForInputable(n, attrMap); !isInputable {
  return strings.Join(content, "\n")
@@ -109,18 +106,12 @@ func (t *HTML2MDTranslator) Visit(n *html.Node) string {
  case "video":
  return "<video>"
  case "a":
- if virtualID, ok := attrMap["data-vid"]; !ok {
+ if !isClickable(n, attrMap) {
  return strings.Join(content, "\n")
- } else if !isClickable(n, attrMap) {
+ } else if label, isClickable := getLabelForClickable(n, attrMap, content); !isClickable {
  return strings.Join(content, "\n")
  } else {
- innerText := parseInnerText(content)
- href, ok := attrMap["href"]
- if !ok {
- return strings.Join(content, "\n")
- }
- strippedQueryParams := stripQueryParamsFromPossibleFullURL(href)
- return renderSelectable(SelectableTypeLink, virtualID, innerText, fmt.Sprintf("href=\"%s\"", strippedQueryParams))
+ return renderSelectable(SelectableTypeLink, virtualID, label, "")
  }
  case "li":
  text := strings.Join(content, "")

translators/html2md/utils.go

@@ -64,25 +64,53 @@ func renderSelectable(typ SelectableType, virtualID string, primaryContent strin
  return fmt.Sprintf("[%s%s, type=%s](%s)", primaryContent, suffix, typ, virtualID)
 }
 
+var whitelistedImportantAttributeValues = map[string][]string{
+ "name": {"login", "search"},
+}
+
 func isClickable(n *html.Node, attrMap map[string]string) bool {
- if n.Data != "a" && n.Data != "button" {
+ if _, ok := attrMap["data-vid"]; !ok {
+ return false
+ } else if n.Data != "a" && n.Data != "button" {
  return false
  } else if ariaHidden, ok := attrMap["aria-hidden"]; ok && ariaHidden == "true" {
  return false
  }
+ if ariaLabel, ok := attrMap["aria-label"]; ok && ariaLabel != "" {
+ return true
+ } else if name, ok := attrMap["name"]; ok && name != "" {
+ if importantNameValues, ok := whitelistedImportantAttributeValues["name"]; ok {
+ if slicesx.Contains(importantNameValues, name) {
+ return true
+ }
+ }
+ }
 
+ if typ, ok := attrMap["type"]; ok && typ == "submit" {
+ return true
+ }
  if n.Data == "a" {
  if href, ok := attrMap["href"]; ok && href != "" {
  return true
+ } else if role, ok := attrMap["role"]; ok && role == "button" {
+ return true
+ } else if name, ok := attrMap["name"]; ok && name == "login" {
+ return true
  }
  } else if n.Data == "button" {
- return true
+ if ariaExpanded, ok := attrMap["aria-expanded"]; ok {
+ return ariaExpanded == "true" || ariaExpanded == "false"
+ } else if inForm := isInForm(n); inForm {
+ return true
+ }
  }
  return false
 }
 
 func isInputable(n *html.Node, attrMap map[string]string) bool {
- if n.Data != "input" && n.Data != "textarea" {
+ if _, ok := attrMap["data-vid"]; !ok {
+ return false
+ } else if n.Data != "input" && n.Data != "textarea" {
  return false
  } else if typ, ok := attrMap["type"]; !ok || typ == "hidden" {
  return false
@@ -116,6 +144,49 @@ func isInputable(n *html.Node, attrMap map[string]string) bool {
  return false
 }
 
+// TODO: this is currently more conservative than isClickable
+func getLabelForClickable(n *html.Node, attrMap map[string]string, childContent []string) (label string, isClickable bool) {
+ if n.Data != "a" && n.Data != "button" {
+ return "", false
+ } else if ariaLabel, ok := attrMap["aria-label"]; ok && ariaLabel != "" {
+ return ariaLabel, true
+ }
+ innerText := parseInnerText(childContent)
+ var importantAttributePairs []string
+ var prefix string
+ for attr, values := range whitelistedImportantAttributeValues {
+ if value, ok := attrMap[attr]; ok && slicesx.Contains(values, value) {
+ importantAttributePairs = append(importantAttributePairs, fmt.Sprintf("%s=%s", attr, value))
+ }
+ }
+ if len(importantAttributePairs) > 0 {
+ prefix = fmt.Sprintf("%s, ", strings.Join(importantAttributePairs, ", "))
+ }
+ if n.Data == "a" {
+ href, ok := attrMap["href"]
+ if !ok {
+ return "", false
+ }
+ strippedQueryParams := stripQueryParamsFromPossibleFullURL(href)
+ if innerText == "" {
+ return fmt.Sprintf("%shref=%s", prefix, strippedQueryParams), true
+ } else {
+ return fmt.Sprintf("%sinner-text=%s, href=%s", prefix, innerText, strippedQueryParams), true
+ }
+ }
+ if typ, ok := attrMap["type"]; (!ok || typ != "submit") && innerText == "" {
+ return "", false
+ } else {
+ if typ == "submit" {
+ prefix = prefix + "type=submit"
+ }
+ if innerText != "" {
+ prefix = prefix + "inner-text=" + innerText
+ }
+ return strings.TrimRight(prefix, ", "), true
+ }
+}
+
 // TODO: this is currently more conservative than isInputable
 func getLabelForInputable(n *html.Node, attrMap map[string]string) (label string, isInputable bool) {
  if n.Data != "input" && n.Data != "textarea" {
@@ -130,6 +201,16 @@ func getLabelForInputable(n *html.Node, attrMap map[string]string) (label string
  return "", false
 }
 
+func isInForm(n *html.Node) bool {
+ if n.Data == "form" {
+ return true
+ }
+ if n.Parent == nil {
+ return false
+ }
+ return isInForm(n.Parent)
+}
+
 func cleanup(mdText string) string {
  // remove extra newlines
  s := stringsx.ReduceNewlines(mdText, 2)