From 4584b0fd1d6c6b59e14345118ce1b54faa622703 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Fri, 8 Oct 2021 15:44:08 +0200 Subject: [PATCH 1/3] Add no user option --- input.py | 7 +++++++ twitterbot2.py | 29 +++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/input.py b/input.py index 12c0420..164093b 100644 --- a/input.py +++ b/input.py @@ -44,6 +44,13 @@ def get_args(): help="Search for tweets with a defined keyword.", ) + parser.add_argument( + "-nu", + "--no-user", + action="store_true", + help="Don't like and retweet user tweets.", + ) + group.add_argument( "-s", "--stats", diff --git a/twitterbot2.py b/twitterbot2.py index cbc9919..e3a89e3 100644 --- a/twitterbot2.py +++ b/twitterbot2.py @@ -176,7 +176,7 @@ def likes_rt_user(bot, logger, tweet_count, likes_count, retweet_count): return tweet_count, likes_count, retweet_count -def crawl_timeline(bot, logger): +def crawl_timeline(bot, logger, no_user): """ This is the handle function of the -t or --timeline option. """ @@ -219,9 +219,14 @@ def crawl_timeline(bot, logger): logger.info("Sleeping for one minute.") time.sleep(60) - tweet_count, likes_count, retweet_count = likes_rt_user( - bot, logger, tweet_count, likes_count, retweet_count - ) + if no_user: + tweet_count, likes_count, retweet_count = likes_rt_home( + bot, logger, tweet_count, likes_count, retweet_count + ) + else: + tweet_count, likes_count, retweet_count = likes_rt_user( + bot, logger, tweet_count, likes_count, retweet_count + ) # update the values in the database today = datetime.datetime.today().strftime("%Y-%m-%d") @@ -290,7 +295,7 @@ def likes_rt_search(bot, logger, keyword, tweet_count, likes_count, retweet_coun return tweet_count, likes_count, retweet_count -def crawl_keyword(bot, logger, keyword): +def crawl_keyword(bot, logger, keyword, no_user): """ This is the handle function of the -k or --keyword option. """ @@ -330,12 +335,18 @@ def crawl_keyword(bot, logger, keyword): tweet_count, likes_count, retweet_count = likes_rt_search( bot, logger, keyword, tweet_count, likes_count, retweet_count ) + logger.info("Sleeping for one minute.") time.sleep(60) - tweet_count, likes_count, retweet_count = likes_rt_user( - bot, logger, tweet_count, likes_count, retweet_count - ) + if no_user: + tweet_count, likes_count, retweet_count = likes_rt_search( + bot, logger, keyword, tweet_count, likes_count, retweet_count + ) + else: + tweet_count, likes_count, retweet_count = likes_rt_user( + bot, logger, tweet_count, likes_count, retweet_count + ) # update the values in the database today = datetime.datetime.today().strftime("%Y-%m-%d") @@ -438,6 +449,7 @@ def main(): args=( bot, logger, + args.no_user, ), ) t2 = Thread(target=server.app.run, kwargs={"host": "0.0.0.0"}) @@ -454,6 +466,7 @@ def main(): bot, logger, args.keyword, + args.no_user, ), ) t2 = Thread(target=server.app.run, kwargs={"host": "0.0.0.0"}) From ec5a80aea7e964bbdb47c7da0516b1a2de100762 Mon Sep 17 00:00:00 2001 From: edoardottt Date: Fri, 8 Oct 2021 15:53:47 +0200 Subject: [PATCH 2/3] Add no user option --- usage.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/usage.py b/usage.py index 95aaffd..447ef98 100644 --- a/usage.py +++ b/usage.py @@ -14,7 +14,7 @@ def usage(): """ - usage: twitterbot2.py [-h] [-v | -t | -k KEYWORD | -s STATS | -oc OUTPUT_CSV | -oj OUTPUT_JSON | -oh OUTPUT_HTML] + usage: twitterbot2.py [-h] [-v | -t | -k KEYWORD | -nu | -s STATS | -oc OUTPUT_CSV | -oj OUTPUT_JSON | -oh OUTPUT_HTML] Twitterbot v2 @@ -24,6 +24,7 @@ def usage(): -t, --timeline Search for tweets in the bot and user's timeline. -k KEYWORD, --keyword KEYWORD Search for tweets with a defined keyword. + -nu, --no-user Don't like and retweet user tweets. -s STATS, --stats STATS Show the statistics of the inputted bot (username). -oc OUTPUT_CSV, --output-csv OUTPUT_CSV @@ -43,6 +44,7 @@ def usage(): print(" -t, --timeline Search for tweets in the bot and user's timeline.") print(" -k KEYWORD, --keyword KEYWORD") print(" Search for tweets with a defined keyword.") + print(" -nu, --no-user Don't like and retweet user tweets.") print(" -s STATS, --stats STATS") print(" Show the statistics of the inputted bot (username).") print(" -oc OUTPUT_CSV, --output-csv OUTPUT_CSV") From b333fd020a3635a085985258c756c29acdf4b1de Mon Sep 17 00:00:00 2001 From: edoardottt Date: Wed, 13 Oct 2021 16:57:10 +0200 Subject: [PATCH 3/3] urllib error handling --- README.md | 7 ++++--- errors.py | 41 ++++++++++++++++++++++++++--------------- twitterbot2.py | 16 ++++++++-------- 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 6600ba8..dce35ed 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Description 🔦 -------- -Simple twitter bot that can like and retweet your tweets, or search tweets with a specified topic. It stores and serves the data with a Flask webapp. 🐦 +Simple twitter bot that can like and retweet your tweets, or search tweets with a specified topic. It stores and serves data with a Flask webapp. 🐦 Installation 📡 ------- @@ -44,7 +44,7 @@ Usage 🚀 ------- ``` -usage: twitterbot2.py [-h] [-v | -t | -k KEYWORD | -s STATS | -oc OUTPUT_CSV | -oj OUTPUT_JSON | -oh OUTPUT_HTML] +usage: twitterbot2.py [-h] [-v] [-t] [-k KEYWORD] [-nu] [-s STATS] [-oc OUTPUT_CSV] [-oj OUTPUT_JSON] [-oh OUTPUT_HTML] Twitterbot v2 @@ -54,6 +54,7 @@ optional arguments: -t, --timeline Search for tweets in the bot and user's timeline. -k KEYWORD, --keyword KEYWORD Search for tweets with a defined keyword. + -nu, --no-user Don't like and retweet user tweets. -s STATS, --stats STATS Show the statistics of the inputted bot (username). -oc OUTPUT_CSV, --output-csv OUTPUT_CSV @@ -69,7 +70,7 @@ Useful notes/links 🔗 - [Getting access to the Twitter api](https://developer.twitter.com/en/docs/twitter-api/getting-started/getting-access-to-the-twitter-api) - [Twitter api rate limits](https://developer.twitter.com/en/docs/twitter-api/v1/rate-limits) -- This app uses a Flask server. Do not expose this on the public Internet, use this behind NAT/firewall. +- This app uses a Flask webserver. Don't expose it on public Internet, use this behind NAT/firewall. Contributing 🤝 -------- diff --git a/errors.py b/errors.py index 4f91b11..e211a9e 100644 --- a/errors.py +++ b/errors.py @@ -14,30 +14,41 @@ import time import logging +import twitter +import urllib def error_handler(e): """ - This function handles all the TwitterHTTPError errors + This function handles all the twitterbot2 errors + - twitter.api.TwitterHTTPError + - urllib.error.URLError + - generic """ + logger = logging.getLogger("__main__") - logger.error(str(e.e) + " on " + e.uri) + if e.__class__ == twitter.api.TwitterHTTPError: + logger.error(str(e.e) + " on " + e.uri) - # == 429 TOO MANY REQUESTS -> Sleep for one hour - if str(e.e.code) == "429": - logger.info("Sleeping for one hour.") - time.sleep(60 * 60) - return + # == 429 TOO MANY REQUESTS -> Sleep for one hour + if str(e.e.code) == "429": + logger.info("Sleeping for one hour.") + time.sleep(60 * 60) + return - # == 403 FORBIDDEN -> Sleep for ten seconds - if str(e.e.code) == "403": - logger.info("Sleeping for ten seconds.") - time.sleep(10) - return + # == 403 FORBIDDEN -> Sleep for ten seconds + if str(e.e.code) == "403": + logger.info("Sleeping for ten seconds.") + time.sleep(10) + return + elif e.__class__ == urllib.error.URLError: + logger.error(str(e.reason)) + logger.info("Sleeping for five minutes.") + time.sleep(5 * 60) # for other types of issues with specific behaviour # add here an additional handler - # == DEFAULT == - logger.info("Sleeping for ten seconds.") - time.sleep(10) + else: + logger.info("Sleeping for ten seconds.") + time.sleep(10) diff --git a/twitterbot2.py b/twitterbot2.py index e3a89e3..ad8b3a5 100644 --- a/twitterbot2.py +++ b/twitterbot2.py @@ -121,7 +121,7 @@ def likes_rt_home(bot, logger, tweet_count, likes_count, retweet_count): try: home = get_home(bot) - except twitter.api.TwitterHTTPError as e: + except Exception as e: errors.error_handler(e) if home is not None: @@ -132,12 +132,12 @@ def likes_rt_home(bot, logger, tweet_count, likes_count, retweet_count): try: likes_count = put_like(bot, tweet_home, logger, likes_count) - except twitter.api.TwitterHTTPError as e: + except Exception as e: errors.error_handler(e) try: retweet_count = retweet_tweet(bot, tweet_home, logger, retweet_count) - except twitter.api.TwitterHTTPError as e: + except Exception as e: errors.error_handler(e) time.sleep(2) @@ -152,7 +152,7 @@ def likes_rt_user(bot, logger, tweet_count, likes_count, retweet_count): """ try: home = get_friend_home(bot, globals.user) - except twitter.api.TwitterHTTPError as e: + except Exception as e: errors.error_handler(e) if home is not None: @@ -163,12 +163,12 @@ def likes_rt_user(bot, logger, tweet_count, likes_count, retweet_count): try: likes_count = put_like(bot, tweet_home, logger, likes_count) - except twitter.api.TwitterHTTPError as e: + except Exception as e: errors.error_handler(e) try: retweet_count = retweet_tweet(bot, tweet_home, logger, retweet_count) - except twitter.api.TwitterHTTPError as e: + except Exception as e: errors.error_handler(e) time.sleep(2) @@ -235,7 +235,7 @@ def crawl_timeline(bot, logger, no_user): # retrieve the up-to-date followers count try: followers_count = followers(bot, globals.bot_user) - except twitter.api.TwitterHTTPError as e: + except Exception as e: errors.error_handler(e) # if there aren't data, creates a record in the statistics table @@ -390,7 +390,7 @@ def crawl_keyword(bot, logger, keyword, no_user): time.sleep(15 * 60) logger.info("Sleeping for 15 minutes.") time.sleep(15 * 60) - except twitter.api.TwitterHTTPError as e: + except Exception as e: if tweet_count != 0: db.update_stat(