feat: support for 2024-09-06 step

Author: kjellberg

lib/serp_parser/google/organic_results/featured_result1.rb

@@ -0,0 +1,80 @@
+module SerpParser
+  module Google
+    module OrganicResults
+      class FeaturedResult1 < SerpParser::Google::Search
+        include SerpParser::Google::OrganicResults::Shared
+        include SerpParser::Helpers
+
+        # @return [String]
+        SELECTOR = "div.Gx5Zad.xpd.EtOod.pkphOe"
+        REQUIRED_CHILDREN = ["span div.BNeawe span.rQMQod.Xb5VRe"]
+
+        # List of allowed span elements (determined by class name) in the description.
+        # @return [Array]
+        ALLOWED_DESCRIPTION_ELEMENTS = [ "r0bn4c rQMQod" ]
+
+        # The schema for the organic result.
+        # @return [Hash]
+        SCHEMA = {
+          title: {
+            type: :instance_method
+          },
+          description: {
+            type: :instance_method
+          },
+          url: {
+            type: :instance_method
+          }, 
+          site_links: {
+            type: :collection,
+            parsers: [
+              SerpParser::Google::OrganicResults::SiteLinks::SiteLink2
+            ]
+          },
+          rating: {
+            type: :hash,
+            parsers: [
+              SerpParser::Google::OrganicResults::Ratings::Rating2
+            ]
+          }
+        }
+
+        # Returns the title of the result
+        # @return [String]
+        def title
+          element = @doc.css("span.rQMQod.Xb5VRe")
+          clean_text element.text
+        end
+
+        # Returns the URL of the result
+        # @return [String]
+        def url
+          element = @doc.css(".kCrYT a")
+          clean_google_url element.first["href"]
+        end
+        
+        # Returns the description of the result
+        # @return [String]
+        def description
+          element = @doc.css(".PqksIc.nRlVm")
+          return if element.nil?
+
+          clean_text element.text
+        end
+
+        private
+
+        # Find the text node that contains the description text
+        # @param element [Nokogiri::XML::Element]
+        # @return [Nokogiri::XML::Element]
+        def find_description_text_node(element)
+          element.reverse_each.find do |node|
+            node.children.all? do |child|
+              child.text? || (child.element? && child.name == "span" && (child["class"].nil? || child["class"].strip.empty? || ALLOWED_DESCRIPTION_ELEMENTS.include?(child["class"])))
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/serp_parser/google/organic_results/organic_result1.rb

@@ -7,6 +7,7 @@ class OrganicResult1 < SerpParser::Google::Search
 
         # @return [String]
         SELECTOR = "div.g.Ww4FFb"
+        REQUIRED_CHILDREN = ["h3"]
 
         # The schema for the organic result.
         # @return [Hash]

lib/serp_parser/google/organic_results/organic_result2.rb

@@ -6,7 +6,8 @@ class OrganicResult2 < SerpParser::Google::Search
         include SerpParser::Helpers
 
         # @return [String]
-        SELECTOR = "div.Gx5Zad.fP1Qef.xpd.EtOod.pkphOe"
+        SELECTOR = "div.Gx5Zad.xpd.EtOod.pkphOe"
+        REQUIRED_CHILDREN = ["h3"]
 
         # List of allowed span elements (determined by class name) in the description.
         # @return [Array]
@@ -59,6 +60,8 @@ def description
           element = @doc.css(".BNeawe.s3v9rd.AP7Wnd")
           element = find_description_text_node(element)
           element = remove_span_elements(element)
+          return if element.nil?
+
           clean_text element.text
         end
 

lib/serp_parser/google/search.rb

@@ -6,7 +6,9 @@ class Search < SerpParser::Parser
         organic_results: {
           type: :collection,
           parsers: [
-            SerpParser::Google::OrganicResults::OrganicResult1
+            SerpParser::Google::OrganicResults::FeaturedResult1,
+            SerpParser::Google::OrganicResults::OrganicResult1,
+            SerpParser::Google::OrganicResults::OrganicResult2,
           ]
         },
         search_information: {

lib/serp_parser/parser.rb

@@ -68,8 +68,16 @@ def build_merged_hash(parsers, default_values: {})
     def parse_children(parsers)
       parsers.flat_map do |parser|
         @doc.css(parser::SELECTOR).map do |element|
+          if defined?(parser::REQUIRED_CHILDREN) && parser::REQUIRED_CHILDREN.any?
+            # Check if the element contains the required child elements (direct children only)
+            required_children_exist = parser::REQUIRED_CHILDREN.all? do |child_selector|
+              !element.children.css(child_selector).empty?
+            end
+            next unless required_children_exist
+          end
+
           parser.new(element).processed_data
-        end
+        end.compact  # Remove nil values resulting from the `next` statement
       end
     end
 

spec/files/google/2024-09-06.html

@@ -0,0 +1 @@
+<div><div class="Gx5Zad xpd EtOod pkphOe"><div class="egMi0 kCrYT"><a href="/url?q=https://middagskassen.se/stad/goteborg/&amp;sa=U&amp;ved=2ahUKEwiuhOiSpq6IAxWKpZUCHY0LByIQFnoECGIQAg&amp;usg=AOvVaw29sektXHtJGqYYCXhPYQOh" data-ved="2ahUKEwiuhOiSpq6IAxWKpZUCHY0LByIQFnoECGIQAg"><div class="DnJfK"><div class="j039Wc"><h3 class="zBAuLc l97dzf"><div class="BNeawe vvjwJb AP7Wnd UwRFLe" style="-webkit-line-clamp:2">Hela listan (2024). - Matkassar i Göteborg med hemleverans</div></h3></div><div class="sCuL3"><div class="BNeawe UPmit AP7Wnd lRVwie">middagskassen.se › Städer</div></div></div></a></div><div class="kCrYT"><div><div class="BNeawe s3v9rd AP7Wnd"><div><div><div class="BNeawe s3v9rd AP7Wnd">Lista med leverantörer som kör ut matkassar till adresser i Göteborg och nära orter. ✓Rabattkoder. ✓Omdömen. ✓Bäst i Test 2024.</div></div></div></div></div></div></div></div>

spec/files/google/organic_result3/general_result.html

@@ -0,0 +1,8 @@
+{
+  "title": "Hela listan (2024). - Matkassar i Göteborg med hemleverans",
+  "description": "Lista med leverantörer som kör ut matkassar till adresser i Göteborg och nära orter. ✓Rabattkoder. ✓Omdömen. ✓Bäst i Test 2024.",
+  "url": "https://middagskassen.se/stad/goteborg/",
+  "date": null,
+  "rating": null,
+  "site_links": []
+}

spec/files/google/organic_result3/general_result.json

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+require "json"
+
+RSpec.describe SerpParser::Google::OrganicResults::OrganicResult2 do
+  it_behaves_like "matches the expected output", "google/organic_result3/general_result"
+end

spec/serp_parser/google/organic_result3_spec.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+RSpec.describe SerpParser::Google::Search do
+  let(:html) { File.read("spec/files/google/2024-09-06.html") }
+  let(:doc) { Nokogiri::HTML::DocumentFragment.parse(html) }
+  let(:parser) { described_class.new(html) }
+  let(:parser_with_html) { described_class.new(html) }
+
+  describe "#initialize" do
+    it "initializes with doc" do
+      expect { parser }.not_to raise_error
+    end
+
+    it "initializes with html" do
+      expect { parser_with_html }.not_to raise_error
+    end
+  end
+
+  describe "#search_information" do
+    it "returns hash with information" do
+      expect(parser.search_information).to be_an_instance_of(Hash)
+    end
+  end
+
+  describe "#organic_results" do
+    it "returns a collection object" do
+      expect(parser.organic_results).to be_an_instance_of(SerpParser::Collection)
+    end
+
+    it "returns organic results" do
+      expect(parser.organic_results).to all(be_an_instance_of(SerpParser::Models::OrganicResult))
+    end
+
+    it "returns 98 organic results" do
+      expect(parser.organic_results.size).to eq(99)
+    end
+
+    it "returns featured post as first organic result" do
+      expect(parser.organic_results.first.title).to eq("Matkasse Göteborg | Jämför de bästa matkassarna 2024")
+      expect(parser.organic_results.first.description).to match("Bästa matkassarna i Göteborg")
+      expect(parser.organic_results.first.description).to match("Beställ från HelloFresh.")
+      expect(parser.organic_results.first.url).to eq("https://matkassen.nu/matkasse-goteborg/")
+    end
+
+    describe "#site_links of first result" do
+      it "returns an array of site links" do
+        expect(parser.organic_results[9].site_links).to all(be_an_instance_of(SerpParser::Models::OrganicResults::SiteLink))
+      end
+
+      it "returns 4 site links" do
+        expect(parser.organic_results[9].site_links.size).to eq(4)
+      end
+    end
+  end
+end