fix(cli): exclude hidden elements from scrape command output

Fixes bounty issue #1707

The scrape command was including text from hidden DOM elements in its
output. This fix adds visibility checks before processing elements to
skip content that is hidden via:
- hidden attribute
- style="display:none"
- style="visibility:hidden"
- aria-hidden="true"

Author: factorydroid

cortex-cli/src/scrape_cmd.rs

@@ -304,6 +304,11 @@ fn process_node_to_markdown(
             scraper::Node::Element(elem) => {
                 let tag = elem.name.local.as_ref();
                 if let Some(element_ref) = scraper::ElementRef::wrap(child) {
+                    // Skip hidden elements
+                    if is_element_hidden(elem) {
+                        continue;
+                    }
+
                     match tag {
                         // Skip unwanted elements
                         "script" | "style" | "nav" | "footer" | "header" | "aside" | "noscript" => {
@@ -683,6 +688,11 @@ fn extract_text_content(node: scraper::ElementRef, output: &mut String) {
             scraper::Node::Element(elem) => {
                 let tag = elem.name.local.as_ref();
                 if let Some(element_ref) = scraper::ElementRef::wrap(child) {
+                    // Skip hidden elements
+                    if is_element_hidden(elem) {
+                        continue;
+                    }
+
                     match tag {
                         // Skip unwanted elements
                         "script" | "style" | "nav" | "footer" | "header" | "aside" | "noscript" => {
@@ -830,6 +840,35 @@ fn unwrap_tags(html: &str, tag: &str) -> String {
     output
 }
 
+/// Check if an HTML element is hidden via various visibility attributes.
+/// Returns true if the element should be skipped during content extraction.
+fn is_element_hidden(elem: &scraper::node::Element) -> bool {
+    // Check for hidden attribute
+    if elem.attr("hidden").is_some() {
+        return true;
+    }
+
+    // Check for aria-hidden="true"
+    if elem.attr("aria-hidden") == Some("true") {
+        return true;
+    }
+
+    // Check for style attribute containing visibility-hiding properties
+    if let Some(style) = elem.attr("style") {
+        let style_lower = style.to_lowercase();
+        // Check for display:none (with or without spaces)
+        if style_lower.contains("display:none") || style_lower.contains("display: none") {
+            return true;
+        }
+        // Check for visibility:hidden (with or without spaces)
+        if style_lower.contains("visibility:hidden") || style_lower.contains("visibility: hidden") {
+            return true;
+        }
+    }
+
+    false
+}
+
 /// Normalize whitespace in text.
 fn normalize_whitespace(text: &str) -> String {
     let mut result = String::new();
@@ -1003,4 +1042,59 @@ mod tests {
         let result2 = normalize_whitespace("no\n\nextra\t\tspaces");
         assert!(result2.contains("no") && result2.contains("extra") && result2.contains("spaces"));
     }
+
+    #[test]
+    fn test_hidden_elements_excluded_from_markdown() {
+        // Test hidden attribute
+        let html = r#"<p>Visible</p><p hidden>Hidden by attribute</p>"#;
+        let md = html_to_markdown(html, false, false);
+        assert!(md.contains("Visible"));
+        assert!(!md.contains("Hidden by attribute"));
+
+        // Test style="display:none"
+        let html = r#"<p>Visible</p><p style="display:none">Hidden by display none</p>"#;
+        let md = html_to_markdown(html, false, false);
+        assert!(md.contains("Visible"));
+        assert!(!md.contains("Hidden by display none"));
+
+        // Test style="display: none" (with space)
+        let html =
+            r#"<p>Visible</p><p style="display: none">Hidden by display none with space</p>"#;
+        let md = html_to_markdown(html, false, false);
+        assert!(md.contains("Visible"));
+        assert!(!md.contains("Hidden by display none with space"));
+
+        // Test style="visibility:hidden"
+        let html = r#"<p>Visible</p><p style="visibility:hidden">Hidden by visibility</p>"#;
+        let md = html_to_markdown(html, false, false);
+        assert!(md.contains("Visible"));
+        assert!(!md.contains("Hidden by visibility"));
+
+        // Test aria-hidden="true"
+        let html = r#"<p>Visible</p><p aria-hidden="true">Hidden by aria</p>"#;
+        let md = html_to_markdown(html, false, false);
+        assert!(md.contains("Visible"));
+        assert!(!md.contains("Hidden by aria"));
+    }
+
+    #[test]
+    fn test_hidden_elements_excluded_from_text() {
+        // Test hidden attribute
+        let html = r#"<p>Visible</p><p hidden>Hidden by attribute</p>"#;
+        let text = html_to_text(html);
+        assert!(text.contains("Visible"));
+        assert!(!text.contains("Hidden by attribute"));
+
+        // Test style="display:none"
+        let html = r#"<p>Visible</p><p style="display:none">Hidden by display none</p>"#;
+        let text = html_to_text(html);
+        assert!(text.contains("Visible"));
+        assert!(!text.contains("Hidden by display none"));
+
+        // Test aria-hidden="true"
+        let html = r#"<p>Visible</p><p aria-hidden="true">Hidden by aria</p>"#;
+        let text = html_to_text(html);
+        assert!(text.contains("Visible"));
+        assert!(!text.contains("Hidden by aria"));
+    }
 }