重开仓库内容：改推五个技能（browser-harness / humanizer / humanizer-zh / product-planning / session-mechanism）
按授权清空原有内容后重新提交（原 oil-ui-pro 一并移出，可从历史恢复）。 browser-harness 剔除 .venv 等运行环境；根 .gitignore 补记 .venv/ 与 node_modules/。
This commit is contained in:
admin committed
1 parent
9a152a1952
commit
777f7fe5d0
240 files changed
+52518
-4409
No files matched your search
@@ -0,0 +1,375 @@
|
||||
# Trustpilot — Company Reviews Scraping
|
||||
|
||||
Field-tested against trustpilot.com on 2026-04-18.
|
||||
`http_get` with a generic Mozilla/5.0 UA works — no JS challenge, no Cloudflare block.
|
||||
The Trustpilot Consumer API (`api.trustpilot.com`) returns 403 for all endpoints without an API key.
|
||||
|
||||
---
|
||||
|
||||
## Fastest Approach: `http_get` + `__NEXT_DATA__`
|
||||
|
||||
Trustpilot is a Next.js SSR app. Every company review page embeds the full data payload in a
|
||||
`<script id="__NEXT_DATA__">` JSON block — no browser needed. This includes the business unit
|
||||
metadata, all 20 reviews for the current page, pagination info, and rating distribution.
|
||||
|
||||
```python
|
||||
import re, json
|
||||
from helpers import http_get
|
||||
|
||||
def get_trustpilot_page(domain, page=1, stars=None, languages='en', verified=False):
|
||||
"""
|
||||
Fetch one page of reviews for a company domain.
|
||||
Returns (business_unit, reviews, pagination, rating_distribution).
|
||||
Returns (None, [], {}, {}) if page is beyond the cap or no data.
|
||||
"""
|
||||
url = f"https://www.trustpilot.com/review/{domain}?languages={languages}&page={page}"
|
||||
if stars:
|
||||
url += f"&stars={stars}"
|
||||
if verified:
|
||||
url += "&verified=true"
|
||||
|
||||
html = http_get(url)
|
||||
m = re.search(
|
||||
r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
|
||||
html, re.DOTALL
|
||||
)
|
||||
if not m:
|
||||
return None, [], {}, {}
|
||||
|
||||
data = json.loads(m.group(1))
|
||||
pp = data['props']['pageProps']
|
||||
bu = pp['businessUnit']
|
||||
filters = pp.get('filters') or {}
|
||||
pagination = filters.get('pagination', {})
|
||||
ratings = filters.get('reviewStatistics', {}).get('ratings', {})
|
||||
reviews = pp.get('reviews', [])
|
||||
|
||||
return bu, reviews, pagination, ratings
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Business Unit (Company) Metadata
|
||||
|
||||
```python
|
||||
bu, reviews, pagination, ratings = get_trustpilot_page("amazon.com")
|
||||
|
||||
# Confirmed fields (tested 2026-04-18):
|
||||
bu['id'] # '46ad346800006400050092d0' — stable MongoDB ObjectId
|
||||
bu['displayName'] # 'Amazon'
|
||||
bu['identifyingName'] # 'www.amazon.com'
|
||||
bu['trustScore'] # 1.7 (float, 1.0–5.0)
|
||||
bu['stars'] # 1.5 (display stars: 1, 1.5, 2, 2.5 … 5)
|
||||
bu['numberOfReviews'] # 45228 — total across all languages
|
||||
bu['websiteUrl'] # 'https://www.amazon.com'
|
||||
bu['isClaimed'] # True/False
|
||||
bu['isClosed'] # True/False
|
||||
bu['isCollectingReviews'] # True/False
|
||||
|
||||
# Rating distribution (from filters.reviewStatistics.ratings):
|
||||
ratings # {'total': 45228, 'one': 29718, 'two': 2701, 'three': 1759, 'four': 2367, 'five': 8683}
|
||||
|
||||
# Pagination (filtered count, default is English only):
|
||||
pagination # {'currentPage': 1, 'perPage': 20, 'totalCount': 28039, 'totalPages': 1402}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Review Fields
|
||||
|
||||
Each review in the `reviews` list has these confirmed fields:
|
||||
|
||||
```python
|
||||
review = {
|
||||
'id': '69e3103e09f46d6b5910f3c1', # hex ObjectId, unique
|
||||
'rating': 1, # int 1–5
|
||||
'title': 'UNDELIVERABLE',
|
||||
'text': 'UNDELIVERABLE\nThis is the only explanation...',
|
||||
'language': 'en',
|
||||
'likes': 0, # upvote count
|
||||
'source': 'Organic', # 'Organic' or 'Invitation'
|
||||
'filtered': False,
|
||||
'isPending': False,
|
||||
|
||||
'dates': {
|
||||
'experiencedDate': '2026-03-29T00:00:00.000Z', # when they used the service
|
||||
'publishedDate': '2026-04-18T07:01:50.000Z', # when review was posted
|
||||
'updatedDate': None,
|
||||
'submittedDate': None,
|
||||
},
|
||||
|
||||
'consumer': {
|
||||
'id': '5cafe2feb158a8533b443467',
|
||||
'displayName': 'Baldy Bloke',
|
||||
'imageUrl': 'https://user-images.trustpilot.com/...',
|
||||
'numberOfReviews': 17,
|
||||
'countryCode': 'GB',
|
||||
'hasImage': True,
|
||||
'isVerified': False,
|
||||
},
|
||||
|
||||
'labels': {
|
||||
'verification': {
|
||||
'isVerified': False,
|
||||
'verificationLevel': 'not-verified', # or 'verified'
|
||||
'reviewSourceName': 'Organic',
|
||||
'verificationSource': 'invitation',
|
||||
'createdDateTime': '2026-04-18T07:01:50.000Z',
|
||||
'hasDachExclusion': False,
|
||||
},
|
||||
'merged': None,
|
||||
},
|
||||
|
||||
'reply': None, # or {'message': '...', 'publishedDate': '...', 'updatedDate': None}
|
||||
'location': None, # populated for multi-location businesses
|
||||
'productReviews': [], # non-empty for product-level reviews
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Paginating — Collect Up to 200 Reviews
|
||||
|
||||
**Hard cap: pages 1–10 work; page 11+ returns an empty `reviews` array (no error, just empty).**
|
||||
This cap applies per filter combination, so `stars=1` gives 200 reviews, `stars=2` gives another
|
||||
200, etc.
|
||||
|
||||
```python
|
||||
import re, json, time
|
||||
from helpers import http_get
|
||||
|
||||
def collect_reviews(domain, stars=None, languages='en', max_pages=10, delay=0.5):
|
||||
"""
|
||||
Collect up to max_pages*20 = 200 reviews. Returns list of review dicts.
|
||||
stars: 1-5 to filter by rating (None = all)
|
||||
languages: 'en' (default), 'all', or ISO code like 'de'
|
||||
delay: seconds between requests (0.5 is safe; tested 5 rapid reqs with no block)
|
||||
"""
|
||||
base = f"https://www.trustpilot.com/review/{domain}"
|
||||
params = f"?languages={languages}"
|
||||
if stars:
|
||||
params += f"&stars={stars}"
|
||||
|
||||
all_reviews = []
|
||||
seen_ids = set()
|
||||
|
||||
for page in range(1, max_pages + 1):
|
||||
url = f"{base}{params}&page={page}"
|
||||
html = http_get(url)
|
||||
m = re.search(
|
||||
r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
|
||||
html, re.DOTALL
|
||||
)
|
||||
if not m:
|
||||
break
|
||||
data = json.loads(m.group(1))
|
||||
reviews = data['props']['pageProps'].get('reviews', [])
|
||||
if not reviews:
|
||||
break # hit the page 10 cap or truly no more reviews
|
||||
|
||||
new = [r for r in reviews if r['id'] not in seen_ids]
|
||||
seen_ids.update(r['id'] for r in reviews)
|
||||
all_reviews.extend(new)
|
||||
|
||||
if page < max_pages:
|
||||
time.sleep(delay)
|
||||
|
||||
return all_reviews
|
||||
|
||||
|
||||
# Usage — 200 reviews per call:
|
||||
reviews = collect_reviews("shopify.com") # English only, all ratings
|
||||
reviews_1star = collect_reviews("amazon.com", stars=1) # 200 x 1-star reviews
|
||||
reviews_all = collect_reviews("stripe.com", languages='all') # all languages
|
||||
```
|
||||
|
||||
### Maximize unique reviews by sweeping all star ratings
|
||||
|
||||
Since each star filter gives an independent 200-review window, you can collect up to 1,000
|
||||
reviews per company (pages are deduplicated across filters):
|
||||
|
||||
```python
|
||||
all_reviews = {}
|
||||
for stars in range(1, 6):
|
||||
for r in collect_reviews("amazon.com", stars=stars, delay=0.5):
|
||||
all_reviews[r['id']] = r
|
||||
|
||||
print(f"Total unique reviews: {len(all_reviews)}")
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Filters Reference
|
||||
|
||||
All filter params are appended to the base URL `https://www.trustpilot.com/review/{domain}`:
|
||||
|
||||
| Param | Values | Notes |
|
||||
|---|---|---|
|
||||
| `page` | 1–10 | Page 11+ returns empty `reviews` (tested). 20 reviews per page. |
|
||||
| `languages` | `en`, `all`, `de`, `fr`, `it`, `nl`, `sv`, `da`… | Default is `en`. Use `all` for all languages. |
|
||||
| `stars` | `1`, `2`, `3`, `4`, `5` | Filter to that star rating only. Works correctly. |
|
||||
| `verified` | `true` | Returns only invitation-verified reviews. Amazon has only ~21 verified reviews total. |
|
||||
| `date` | `last30days`, `last6months`, `last12months` | Reflected in `filters.selected.date` but data volume unchanged vs no filter — server-side filtering may be best-effort. |
|
||||
| `sort` | `recency`, `highest_rated`, `lowest_rated`, `helpful` | The `sort` param is accepted but **ignored server-side** via SSR — `filters.selected.sort` always returns `recency`. Sort only works in browser JS navigation. |
|
||||
|
||||
---
|
||||
|
||||
## Pagination Object
|
||||
|
||||
```python
|
||||
# From filters.pagination (present on pages 1–10 when data exists):
|
||||
pagination = {
|
||||
'currentPage': 1,
|
||||
'perPage': 20,
|
||||
'totalCount': 28039, # filtered count (e.g. English only)
|
||||
'totalPages': 1402, # math: ceil(totalCount / 20)
|
||||
}
|
||||
|
||||
# NOTE: totalPages can be 1402 but you can only access pages 1–10 (200 reviews).
|
||||
# On page 11+ the reviews list is empty and pagination is absent.
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Rate Limits and Anti-bot
|
||||
|
||||
- **No Cloudflare, no DataDome** — plain HTTP with `Mozilla/5.0` UA works immediately (tested
|
||||
5 rapid requests in <5 seconds without any block).
|
||||
- **No CAPTCHA** observed during any test run.
|
||||
- **No 429 / rate-limit headers** seen on rapid sequential requests.
|
||||
- Safe rate: 0.5s between requests is conservative. Tested 5 consecutive requests at natural
|
||||
speed (0.2–1s each) with no issue.
|
||||
- **robots.txt** has `User-agent: * / Disallow: /` (all paths blocked for unnamed bots) and
|
||||
explicitly blocks `anthropic-ai`, `ClaudeBot`, `Claude-User`, `Claude-SearchBot`, `GPTBot`,
|
||||
`anthropic-ai`, `CCBot`, etc. Despite this, `http_get` with `Mozilla/5.0` UA is not blocked
|
||||
server-side (robots.txt is advisory only). Respect the policy if operating at scale.
|
||||
|
||||
---
|
||||
|
||||
## Consumer API (`api.trustpilot.com`)
|
||||
|
||||
All Consumer API endpoints require an API key (OAuth2 client credentials). Without a key:
|
||||
|
||||
```
|
||||
GET https://api.trustpilot.com/v1/business-units/find?name=amazon.com → 403 Forbidden
|
||||
GET https://api.trustpilot.com/v1/business-units/{id}/reviews → 403 Forbidden
|
||||
```
|
||||
|
||||
The Business Unit ID embedded in `__NEXT_DATA__` (`businessUnit.id`) is the same ID used in the
|
||||
Consumer API, so if you have an API key, you can use it directly without a separate lookup.
|
||||
|
||||
---
|
||||
|
||||
## Gotchas
|
||||
|
||||
1. **Page cap is 10, not `totalPages`**: `filters.pagination.totalPages` may show 1402, but
|
||||
requests for pages 11+ return `reviews: []` silently. The server-rendered SSR cap is
|
||||
hard-coded at page 10 (200 reviews).
|
||||
|
||||
2. **`totalCount` in pagination is language-filtered**: With `languages=en`, `totalCount` is the
|
||||
English-only count (e.g. 28,039 for Amazon). `businessUnit.numberOfReviews` is the true total
|
||||
across all languages (45,228). Use `languages=all` to see the full count in pagination.
|
||||
|
||||
3. **Sort param ignored in SSR**: `?sort=highest_rated` is reflected in `filters.selected.sort`
|
||||
in the JSON but the reviews returned are always `recency`-sorted. Sort only takes effect
|
||||
via browser-side JS navigation.
|
||||
|
||||
4. **Verified filter is narrow**: Amazon has 45,228 reviews but only 21 are `isVerified=True`
|
||||
(verificationLevel = 'verified'). Most reviews are organic/not-verified. Page 1 of
|
||||
`verified=true` shows a misleading `totalCount=28039` — page 2 corrects to `totalCount=21`.
|
||||
|
||||
5. **`date` filter behavior**: The `date` param is reflected in `filters.selected.date` but the
|
||||
total review counts and returned reviews do not visibly change vs no filter in testing. The
|
||||
server may apply it only partially or it may affect ordering rather than filtering.
|
||||
|
||||
6. **`languages=en` is the default** and the server returns it even without the param. Use
|
||||
`languages=all` explicitly to get reviews in all languages.
|
||||
|
||||
7. **No `__NEXT_DATA__` fallback**: Never observed an empty or missing `__NEXT_DATA__` on valid
|
||||
company pages. If absent, the domain may not have a Trustpilot profile — check for a
|
||||
redirect or 404 in the HTML title.
|
||||
|
||||
8. **Stars `1.5` vs `2`**: `businessUnit.stars` uses half-star display values (1.5, 2.0, etc).
|
||||
`businessUnit.trustScore` is the precise float (1.7). Use `trustScore` for numeric comparison.
|
||||
|
||||
---
|
||||
|
||||
## Complete One-Shot Example
|
||||
|
||||
```python
|
||||
import re, json, time
|
||||
from helpers import http_get
|
||||
|
||||
def scrape_trustpilot(domain, max_unique=200):
|
||||
"""
|
||||
Scrape up to max_unique reviews. Returns (company_info, reviews_list).
|
||||
With max_unique=1000, sweeps all 5 star ratings to maximize coverage.
|
||||
"""
|
||||
def _fetch_page(domain, page, stars=None, languages='en'):
|
||||
url = f"https://www.trustpilot.com/review/{domain}?languages={languages}&page={page}"
|
||||
if stars:
|
||||
url += f"&stars={stars}"
|
||||
html = http_get(url)
|
||||
m = re.search(
|
||||
r'<script id="__NEXT_DATA__" type="application/json">(.*?)</script>',
|
||||
html, re.DOTALL
|
||||
)
|
||||
if not m:
|
||||
return None, []
|
||||
d = json.loads(m.group(1))
|
||||
pp = d['props']['pageProps']
|
||||
return pp['businessUnit'], pp.get('reviews', [])
|
||||
|
||||
company_info = None
|
||||
all_reviews = {}
|
||||
|
||||
# First page to get company info
|
||||
bu, reviews = _fetch_page(domain, 1)
|
||||
company_info = {
|
||||
'id': bu['id'],
|
||||
'name': bu['displayName'],
|
||||
'domain': bu['identifyingName'],
|
||||
'trust_score': bu['trustScore'],
|
||||
'stars': bu['stars'],
|
||||
'total_reviews': bu['numberOfReviews'],
|
||||
'is_claimed': bu['isClaimed'],
|
||||
}
|
||||
for r in reviews:
|
||||
all_reviews[r['id']] = r
|
||||
|
||||
if max_unique <= 20:
|
||||
return company_info, list(all_reviews.values())
|
||||
|
||||
# Pages 2–10 (no star filter)
|
||||
for page in range(2, 11):
|
||||
if len(all_reviews) >= max_unique:
|
||||
break
|
||||
_, reviews = _fetch_page(domain, page)
|
||||
if not reviews:
|
||||
break
|
||||
for r in reviews:
|
||||
all_reviews[r['id']] = r
|
||||
time.sleep(0.5)
|
||||
|
||||
# If we want more, sweep by star rating
|
||||
if len(all_reviews) < max_unique and max_unique > 200:
|
||||
for stars in range(1, 6):
|
||||
for page in range(1, 11):
|
||||
if len(all_reviews) >= max_unique:
|
||||
break
|
||||
_, reviews = _fetch_page(domain, page, stars=stars)
|
||||
if not reviews:
|
||||
break
|
||||
for r in reviews:
|
||||
all_reviews[r['id']] = r
|
||||
time.sleep(0.5)
|
||||
|
||||
return company_info, list(all_reviews.values())[:max_unique]
|
||||
|
||||
|
||||
# Run it:
|
||||
company, reviews = scrape_trustpilot("shopify.com", max_unique=200)
|
||||
print(f"{company['name']} — TrustScore {company['trust_score']} — {company['total_reviews']} total reviews")
|
||||
print(f"Collected: {len(reviews)} reviews")
|
||||
print(f"Sample: [{reviews[0]['rating']}★] {reviews[0]['title'][:60]}")
|
||||
```
|
||||
Reference in new issue
Block a user