-
Notifications
You must be signed in to change notification settings - Fork 0
/
tasks.py
46 lines (31 loc) · 1.03 KB
/
tasks.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
# ================
# File contains tasks to be scheduled on Heroku Server
# ================
#
# SCHEDULED TASKS:
# ---------------
# This file is scheduled to run daily on the Heroku server
# It will first check the day of the week and ONLY RUN it's tasks on a MONDAY
# TASK 1: Call the spider to crawl - store the results in the database
# TASK 2: Check for VisitedURLs older than 100 days and delete them from the database
# =============
# IMPORTS
# =============
import datetime
# Local imports
from news_crawler import crawl
from main.models import VisitedURL
# =======================================
# CHECK DAY OF THE WEEK (MONDAY = 0)
# =======================================
if datetime.datetime.today().weekday() == 0:
# =============
# TASK 1 - crawl
# =============
crawl.crawl()
# # =============
# # TASK 2 - Delete old URLs
# # =============
# today = datetime.datetime.now().date()
# date_threshold = today - datetime.timedelta(days=100)
# VisitedURL.objects.filter(date_visited__lte=date_threshold).delete()