-
Notifications
You must be signed in to change notification settings - Fork 4
/
full_saving_code.py
98 lines (88 loc) · 3.77 KB
/
full_saving_code.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
import pandas as pd
import sqlite3
from google_scholar_organic_results import organic_results
from google_scholar_cite_results import cite_results
# Pandas
print("waiting for organic results to save..")
organic_df = pd.DataFrame(data=organic_results())
organic_df.to_csv("google_scholar_organic_results.csv", encoding="utf-8", index=False)
print("waiting for cite results to save..")
cite_df = pd.DataFrame(data=cite_results())
cite_df.to_csv("google_scholar_citation_results.csv", encoding="utf-8", index=False)
# MySQL
conn = sqlite3.connect("google_scholar_results.db")
conn.execute("""CREATE TABLE google_scholar_organic_results (
page_number integer,
position integer,
result_type text,
title text,
link text,
snippet text,
result_id text,
publication_info_summary text,
cited_by_count integer,
cited_by_link text,
cited_by_id text,
total_versions integer,
all_versions_link text,
all_versions_id text,
file_format text,
file_title text,
file_link text)""")
conn.commit()
conn.execute("""CREATE TABLE google_scholar_cite_results (
organic_results_title text,
organic_results_link text,
citation_title text,
citation_link text)""")
conn.commit()
for item in organic_results():
conn.execute("""INSERT INTO google_scholar_organic_results
VALUES (:page_number,
:position,
:result_type,
:title,
:link,
:snippet,
:result_id,
:publication_info_summary,
:cited_by_count,
:cited_by_link,
:cited_by_id,
:total_versions,
:all_versions_link,
:all_versions_id,
:file_format,
:file_title,
:file_link)""",
{"page_number": item["page_number"],
"position": item["position"],
"result_type": item["type"],
"title": item["title"],
"link": item["link"],
"snippet": item["snippet"],
"result_id": item["result_id"],
"publication_info_summary": item["publication_info_summary"],
"cited_by_count": item["cited_by_count"],
"cited_by_link": item["cited_by_link"],
"cited_by_id": item["cited_by_id"],
"total_versions": item["total_versions"],
"all_versions_link": item["all_versions_link"],
"all_versions_id": item["all_versions_id"],
"file_format": item["file_format"],
"file_title": item["file_title"],
"file_link": item["file_link"]})
conn.commit()
for cite_result in cite_results():
conn.execute("""INSERT INTO google_scholar_cite_results
VALUES (:organic_result_title,
:organic_result_link,
:citation_title,
:citation_snippet)""",
{"organic_result_title": cite_result["organic_result_title"],
"organic_result_link": cite_result["organic_result_link"],
"citation_title": cite_result["citation_title"],
"citation_snippet": cite_result["citation_snippet"]})
conn.commit()
conn.close()
print("Saved to SQL Lite database.")