From 28b36b8b3075b551ca95dad9ad559668df7c3d69 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Sun, 12 Jul 2026 09:45:11 -0700 Subject: [PATCH 1/2] Harden saved-place links and coordinates --- docs/ARCHITECTURE.md | 8 +++- src/gmaps_scraper/cli.py | 5 ++ src/gmaps_scraper/parser.py | 4 ++ src/gmaps_scraper/place_scraper.py | 66 ++++++++++++++++++++++--- tests/test_parser.py | 31 ++++++++++++ tests/test_place_scraper.py | 77 ++++++++++++++++++++++++++++++ 6 files changed, 183 insertions(+), 8 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 642b706..45eb2c4 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -32,7 +32,8 @@ Place entries inside saved lists are detected structurally. The coordinate tuple pattern `[null, null, lat, lng]` is the strongest signal for a saved place record. The parser then uses surrounding parent structures to recover name, address, note, favorite status, Google identifiers, ownership metadata, and a -Maps search URL. +direct CID URL for addressless records when available, or a Maps search URL +otherwise. Saved-list output preserves: @@ -86,6 +87,11 @@ These payloads can contain useful values when the rendered DOM is thin, blocked, or only partially available. Because their shape is undocumented, extraction from preview payloads should favor strongly typed signals: coordinates, plus codes, phone-looking fields, postal addresses, and compact address arrays. +When the final canonical `/maps/place/` URL encodes explicit `!3d`/`!4d` pin +coordinates, those coordinates take precedence: preview payloads can include +unrelated nearby entities or booking offers alongside the selected place. An +`@lat,lng` viewport center remains a fallback only when structured coordinates +are unavailable. ## Legacy Payloads diff --git a/src/gmaps_scraper/cli.py b/src/gmaps_scraper/cli.py index 7ede33c..9484a60 100644 --- a/src/gmaps_scraper/cli.py +++ b/src/gmaps_scraper/cli.py @@ -32,6 +32,7 @@ _merge_llm_place_fields, _merge_place_sources, _place_detail_values, + _prefer_resolved_place_url_coordinates, _repair_source_used_llm, _should_use_llm_repair, collect_place_snapshot, @@ -654,6 +655,10 @@ def _scrape_place_for_debug( snapshot["preview"] if isinstance(snapshot.get("preview"), dict) else {}, ) merged_snapshot = _merge_place_sources(dom_snapshot, preview_snapshot) + merged_snapshot = _prefer_resolved_place_url_coordinates( + merged_snapshot, + resolved_url=resolved_url, + ) details = _build_place_details(place_url, resolved_url=resolved_url, snapshot=merged_snapshot) evidence = _build_place_llm_evidence(merged_snapshot) evidence_hash = _hash_evidence(evidence) diff --git a/src/gmaps_scraper/parser.py b/src/gmaps_scraper/parser.py index df09b5f..7eb59cc 100644 --- a/src/gmaps_scraper/parser.py +++ b/src/gmaps_scraper/parser.py @@ -428,6 +428,7 @@ def _extract_places(node: JSONValue) -> list[Place]: address=address, lat=lat, lng=lng, + cid=cid, ), cid=cid, cid_aliases=cid_aliases, @@ -833,7 +834,10 @@ def _build_maps_url( address: str | None, lat: float, lng: float, + cid: str | None, ) -> str: + if address is None and cid is not None: + return f"https://www.google.com/maps?cid={cid}" query = _build_maps_query(name=name, address=address, lat=lat, lng=lng) return f"https://www.google.com/maps/search/?{urlencode({'api': '1', 'query': query})}" diff --git a/src/gmaps_scraper/place_scraper.py b/src/gmaps_scraper/place_scraper.py index 6b34789..5afd090 100644 --- a/src/gmaps_scraper/place_scraper.py +++ b/src/gmaps_scraper/place_scraper.py @@ -1292,6 +1292,10 @@ def _build_place_details_from_snapshot( (preview_snapshot, "preview"), (search_result_snapshot, "search_result"), ) + merged_snapshot = _prefer_resolved_place_url_coordinates( + merged_snapshot, + resolved_url=resolved_url, + ) details = _build_place_details( place_url, resolved_url=resolved_url, @@ -1932,6 +1936,27 @@ def _looks_like_google_maps_place_url(value: str) -> bool: return parsed.path.startswith("/maps/place/") +def _prefer_resolved_place_url_coordinates( + snapshot: Mapping[str, object], + *, + resolved_url: str | None, +) -> dict[str, object]: + coordinates = _extract_canonical_place_coordinates(resolved_url) + if coordinates is None: + return dict(snapshot) + + merged = dict(snapshot) + raw_field_sources = snapshot.get("field_sources") + field_sources = ( + dict(raw_field_sources) if isinstance(raw_field_sources, Mapping) else {} + ) + merged["lat"], merged["lng"] = coordinates + field_sources["lat"] = "resolved_url" + field_sources["lng"] = "resolved_url" + merged["field_sources"] = field_sources + return merged + + def _collect_review_panel_snapshot(page: Any, *, timeout_ms: int) -> dict[str, object]: try: clicked = page.evaluate(_PLACE_REVIEW_TAB_CLICK_JS) @@ -2021,12 +2046,16 @@ def _build_place_details( category_display_en, category_display_en_source, category_display_en_confidence = ( _derive_category_display_en(category, snapshot) ) - lat = _parse_float(snapshot.get("lat")) - if lat is None: - lat = _extract_coordinate_from_url(resolved_url or source_url, index=0) - lng = _parse_float(snapshot.get("lng")) - if lng is None: - lng = _extract_coordinate_from_url(resolved_url or source_url, index=1) + canonical_coordinates = _extract_canonical_place_coordinates(resolved_url) + if canonical_coordinates is not None: + lat, lng = canonical_coordinates + else: + lat = _parse_float(snapshot.get("lat")) + if lat is None: + lat = _extract_coordinate_from_url(resolved_url or source_url, index=0) + lng = _parse_float(snapshot.get("lng")) + if lng is None: + lng = _extract_coordinate_from_url(resolved_url or source_url, index=1) address = _clean_address_text(snapshot.get("address")) or _extract_address_from_lines( combined_lines ) @@ -4256,7 +4285,9 @@ def _reservation_provider_label_from_url(url: str) -> str: def _extract_coordinate_from_url(url: str, *, index: int) -> float | None: - match = re.search(r"@(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?)", url) + match = re.search(r"!3d(-?\d+(?:\.\d+)?)!4d(-?\d+(?:\.\d+)?)", url) + if match is None: + match = re.search(r"@(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?)", url) if match is None: return None try: @@ -4265,6 +4296,27 @@ def _extract_coordinate_from_url(url: str, *, index: int) -> float | None: return None +def _extract_canonical_place_coordinates( + resolved_url: str | None, +) -> tuple[float, float] | None: + if resolved_url is None or not _looks_like_google_maps_place_url(resolved_url): + return None + match = re.search( + r"!3d(-?\d+(?:\.\d+)?)!4d(-?\d+(?:\.\d+)?)", + resolved_url, + ) + if match is None: + return None + try: + lat = float(match.group(1)) + lng = float(match.group(2)) + except ValueError: + return None + if not _valid_coordinates(lat, lng): + return None + return lat, lng + + def _to_bool(value: object) -> bool: return value is True diff --git a/tests/test_parser.py b/tests/test_parser.py index da4885b..a9ac521 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -249,6 +249,37 @@ def test_builds_search_query_url_when_cid_is_missing(self) -> None: "https://www.google.com/maps/search/?api=1&query=Northwind+Cafe%2C+Example+District", ) + def test_builds_direct_cid_url_when_address_is_missing(self) -> None: + runtime_state = copy.deepcopy(["noise", _LIST_NODE]) + place_metadata = runtime_state[1][8][0][1] + assert isinstance(place_metadata, list) + place_metadata[4] = None + + parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) + + self.assertIsNone(parsed.places[0].address) + self.assertEqual(parsed.places[0].cid, _NORTHWIND_CID) + self.assertEqual( + parsed.places[0].maps_url, + f"https://www.google.com/maps?cid={_NORTHWIND_CID}", + ) + + def test_builds_name_query_url_when_address_and_cid_are_missing(self) -> None: + runtime_state = copy.deepcopy(["noise", _LIST_NODE]) + place_metadata = runtime_state[1][8][0][1] + assert isinstance(place_metadata, list) + place_metadata[4] = None + place_metadata[6] = [None] + + parsed = parse_saved_list_artifacts(_LIST_URL, runtime_state=runtime_state) + + self.assertIsNone(parsed.places[0].address) + self.assertIsNone(parsed.places[0].cid) + self.assertEqual( + parsed.places[0].maps_url, + "https://www.google.com/maps/search/?api=1&query=Northwind+Cafe", + ) + def test_uses_fingerprint_not_positive_s2_cell_from_slot_6(self) -> None: runtime_state = copy.deepcopy(["noise", _LIST_NODE]) diff --git a/tests/test_place_scraper.py b/tests/test_place_scraper.py index 76a7b82..2cb1980 100644 --- a/tests/test_place_scraper.py +++ b/tests/test_place_scraper.py @@ -1184,6 +1184,83 @@ def test_build_place_details_prefers_preview_over_search_card_fallback(self) -> assert details.diagnostics is not None self.assertEqual(details.diagnostics.field_sources.get("category"), "preview") + def test_build_place_details_prefers_canonical_resolved_url_coordinates(self) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": ( + "https://www.google.com/maps/place/La+Paix/" + "@50.8500000,4.3600000,17z/" + "data=!4m6!3m5!1s0x0:0x0!8m2!3d50.8511538!4d4.3650878" + ), + "dom": { + "name": "La Paix", + "address": "Rue Royale 103, 1000 Bruxelles, Belgium", + }, + "preview": { + "lat": 50.901778, + "lng": 4.31745, + }, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.lat, 50.8511538) + self.assertEqual(details.lng, 4.3650878) + assert details.diagnostics is not None + self.assertEqual(details.diagnostics.field_sources.get("lat"), "resolved_url") + self.assertEqual(details.diagnostics.field_sources.get("lng"), "resolved_url") + + def test_build_place_details_keeps_preview_coordinates_without_url_coordinates( + self, + ) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": "https://www.google.com/maps/place/La+Paix", + "dom": {"name": "La Paix"}, + "preview": { + "lat": 50.8511538, + "lng": 4.3650878, + }, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.lat, 50.8511538) + self.assertEqual(details.lng, 4.3650878) + assert details.diagnostics is not None + self.assertEqual(details.diagnostics.field_sources.get("lat"), "preview") + self.assertEqual(details.diagnostics.field_sources.get("lng"), "preview") + + def test_build_place_details_keeps_preview_coordinates_with_viewport_only_url( + self, + ) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": ( + "https://www.google.com/maps/place/La+Paix/" + "@50.901778,4.31745,17z" + ), + "dom": {"name": "La Paix"}, + "preview": { + "lat": 50.8511538, + "lng": 4.3650878, + }, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.lat, 50.8511538) + self.assertEqual(details.lng, 4.3650878) + assert details.diagnostics is not None + self.assertEqual(details.diagnostics.field_sources.get("lat"), "preview") + self.assertEqual(details.diagnostics.field_sources.get("lng"), "preview") + def test_build_place_details_backfills_preview_rating_summary_sources(self) -> None: preview = _extract_preview_place_enrichment( (_PREVIEW_FIXTURE_DIR / "rating_review_count.txt").read_text(encoding="utf-8") From c4bd9afb30c4e6478a2272e35e39c6f93428e798 Mon Sep 17 00:00:00 2001 From: Michael Wu Date: Sun, 12 Jul 2026 14:01:48 -0700 Subject: [PATCH 2/2] Validate all URL coordinate fallbacks --- src/gmaps_scraper/place_scraper.py | 44 ++++++++++++++++------------ tests/test_place_scraper.py | 46 ++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+), 18 deletions(-) diff --git a/src/gmaps_scraper/place_scraper.py b/src/gmaps_scraper/place_scraper.py index 5afd090..393b474 100644 --- a/src/gmaps_scraper/place_scraper.py +++ b/src/gmaps_scraper/place_scraper.py @@ -49,6 +49,9 @@ _TITLE_SELECTORS = ("h1.DUwDvf", "h1.lfPIob", "div[role='main'] h1") _TITLE_SELECTOR = ", ".join(_TITLE_SELECTORS) +_CANONICAL_COORDINATE_PATTERN = re.compile( + r"!3d(-?\d+(?:\.\d+)?)!4d(-?\d+(?:\.\d+)?)" +) _PLACE_LLM_PROMPT_VERSION = "gmaps-place-repair-v1" _TRANSLATION_MEMORY = TranslationMemory.default() type PlaceLLMRepairer = Callable[[PlaceLLMRepairRequest], Mapping[str, object] | None] @@ -4284,16 +4287,32 @@ def _reservation_provider_label_from_url(url: str) -> str: return base.replace("-", " ").replace("_", " ").title() -def _extract_coordinate_from_url(url: str, *, index: int) -> float | None: - match = re.search(r"!3d(-?\d+(?:\.\d+)?)!4d(-?\d+(?:\.\d+)?)", url) - if match is None: - match = re.search(r"@(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?)", url) +def _validated_coordinate_match( + match: re.Match[str] | None, +) -> tuple[float, float] | None: if match is None: return None try: - return float(match.group(index + 1)) + lat = float(match.group(1)) + lng = float(match.group(2)) except ValueError: return None + if not _valid_coordinates(lat, lng): + return None + return lat, lng + + +def _extract_coordinate_from_url(url: str, *, index: int) -> float | None: + canonical_match = _CANONICAL_COORDINATE_PATTERN.search(url) + if canonical_match is not None: + coordinates = _validated_coordinate_match(canonical_match) + else: + coordinates = _validated_coordinate_match( + re.search(r"@(-?\d+(?:\.\d+)?),(-?\d+(?:\.\d+)?)", url) + ) + if coordinates is None: + return None + return coordinates[index] def _extract_canonical_place_coordinates( @@ -4301,20 +4320,9 @@ def _extract_canonical_place_coordinates( ) -> tuple[float, float] | None: if resolved_url is None or not _looks_like_google_maps_place_url(resolved_url): return None - match = re.search( - r"!3d(-?\d+(?:\.\d+)?)!4d(-?\d+(?:\.\d+)?)", - resolved_url, + return _validated_coordinate_match( + _CANONICAL_COORDINATE_PATTERN.search(resolved_url) ) - if match is None: - return None - try: - lat = float(match.group(1)) - lng = float(match.group(2)) - except ValueError: - return None - if not _valid_coordinates(lat, lng): - return None - return lat, lng def _to_bool(value: object) -> bool: diff --git a/tests/test_place_scraper.py b/tests/test_place_scraper.py index 2cb1980..c34274a 100644 --- a/tests/test_place_scraper.py +++ b/tests/test_place_scraper.py @@ -1261,6 +1261,52 @@ def test_build_place_details_keeps_preview_coordinates_with_viewport_only_url( self.assertEqual(details.diagnostics.field_sources.get("lat"), "preview") self.assertEqual(details.diagnostics.field_sources.get("lng"), "preview") + def test_build_place_details_rejects_out_of_range_url_coordinates( + self, + ) -> None: + for resolved_url in ( + ( + "https://www.google.com/maps/place/La+Paix/" + "@50.8500000,4.3600000,17z/" + "data=!4m6!3m5!1s0x0:0x0!8m2!3d950.0!4d4.3650878" + ), + "https://www.google.com/maps/place/La+Paix/@950.0,4.3600000,17z", + ): + with self.subTest(resolved_url=resolved_url): + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": resolved_url, + "dom": {"name": "La Paix"}, + "preview": {}, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertIsNone(details.lat) + self.assertIsNone(details.lng) + + def test_build_place_details_keeps_valid_viewport_coordinate_fallback( + self, + ) -> None: + details = _build_place_details_from_snapshot( + "https://www.google.com/maps?cid=4015507283657238879", + snapshot={ + "resolved_url": ( + "https://www.google.com/maps/place/La+Paix/" + "@50.8500000,4.3600000,17z" + ), + "dom": {"name": "La Paix"}, + "preview": {}, + }, + llm_fallback=None, + llm_policy="never", + ) + + self.assertEqual(details.lat, 50.85) + self.assertEqual(details.lng, 4.36) + def test_build_place_details_backfills_preview_rating_summary_sources(self) -> None: preview = _extract_preview_place_enrichment( (_PREVIEW_FIXTURE_DIR / "rating_review_count.txt").read_text(encoding="utf-8")