-
Notifications
You must be signed in to change notification settings - Fork 3
/
safari_books_online_parser.py
73 lines (50 loc) · 2.71 KB
/
safari_books_online_parser.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul 30 19:59:03 2018
@author: Raleigh Littles
"""
import requests
import re
def retrieve_page_contents(url):
r = requests.get(url)
if r.status_code < 300:
return (r.content.decode())
print("Current url: {0} returned invalid status code.".format(url))
raise ValueError('Invalid server response.')
def parse_contents_into_list(text):
regex_string = "/library/cover/[\d]{2,}"
book_id_matches = re.findall(regex_string, text)
book_ids = [match.split('/')[-1] for match in book_id_matches]
print("Current page found {0} book ids.".format(len(book_ids)))
return book_ids
def write_id_list_to_txt_file(id_list, filename):
with open(filename + ".txt", 'a') as txt_file_handler:
txt_file_handler.write("\n".join([str(book_id) for book_id in id_list]))
txt_file_handler.close()
if __name__ == '__main__':
url_dict = {'career-development': 'https://www.safaribooksonline.com/topics/career-development',
'data': 'https://www.safaribooksonline.com/topics/data',
'design': 'https://www.safaribooksonline.com/topics/design',
'hardware': 'https://www.safaribooksonline.com/topics/hardware',
'math_science_engineering': 'https://www.safaribooksonline.com/topics/math-science-engineering',
'other': 'https://www.safaribooksonline.com/topics/other',
'security': 'https://www.safaribooksonline.com/topics/security',
'software-development': 'https://www.safaribooksonline.com/topics/software-development',
'system-administration': 'https://www.safaribooksonline.com/topics/system-administration',
'travel-hobbies': 'https://www.safaribooksonline.com/topics/travel-hobbies',
'web-mobile': 'https://www.safaribooksonline.com/topics/web-mobile',
'business': 'https://www.safaribooksonline.com/topics/business'}
for topic, url in url_dict.items():
# don't expect to see a topic with more than 100 pages of books in it
book_list_for_topic = []
for page_number in range(1, 100):
try:
page_content = retrieve_page_contents(url + "?page={0}".format(page_number))
except ValueError:
break
finally:
book_list = parse_contents_into_list(page_content)
book_list_for_topic.extend(book_list)
print("{0} book ids found for topic: {1}".format(len(book_list_for_topic), topic))
write_id_list_to_txt_file(book_list_for_topic, topic)