-
Notifications
You must be signed in to change notification settings - Fork 0
/
Trustpilot_parser.py
65 lines (44 loc) · 1.71 KB
/
Trustpilot_parser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
import json
import requests
from bs4 import BeautifulSoup
url = "https://scrape.smartproxy.com/v1/tasks?universal="
payload = {
"target": "universal",
"url": "https://www.trustpilot.com/review/panaceafinancial.com",
"headless": "html",
"device_type": "desktop"
}
headers = {
"accept": "application/json",
"content-type": "application/json",
"authorization": "Basic AUTH"
}
def main():
response = requests.post(url, json=payload, headers=headers)
json_data = response.text
parsed_data = json.loads(json_data)
content = parsed_data['results'][0]['content']
# strip scraped content from backslashes
stripped_content = content.replace('\\','')
soup = BeautifulSoup(stripped_content, "html.parser")
# select review text element
review_text = soup.find_all("p", class_="typography_body-l__KUYFJ typography_appearance-default__AAY17 typography_color-black__5LYEn")
# select review date element
review_date = soup.find_all("p", class_="typography_body-m__xgxZ_ typography_appearance-default__AAY17 typography_color-black__5LYEn")
# select review score element
img_tags = soup.select('.star-rating_starRating__4rrcf img')
reviews = []
for p, d, img_tag in zip(review_text, review_date, img_tags):
# create a dictionary for each review
review = {
'text': p.text,
'date': d.text,
'rating': img_tag['alt']
}
# add the review to the list
reviews.append(review)
# dump the output to .json
with open('reviews.json', 'w') as outfile:
json.dump(reviews, outfile)
if __name__ == "__main__":
main()