diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..c6cb596 --- /dev/null +++ b/.coveragerc @@ -0,0 +1,5 @@ +[report] +exclude_lines = + pragma + pragma: no cover + pragma: win32 cover \ No newline at end of file diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..2ba2307 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,13 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: robertoszek +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: robertoszek +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/CHANGELOG.md b/CHANGELOG.md index fd04e59..5bd7ce6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +## [1.0.0] 16-01-2022 +## Fixed +- `max_tweets` not accepting values higher than 100 +- Video: Not getting the best bitrate version of video attachments in some cases +- Polls: not being retrieved for accounts with protected tweets +- RTs: not getting original tweet media attachments +## Added +- `twitter_username` value can be a list now, for having multiple Twitter accounts as sources for one target Fediverse account. +- A Twitter [archive](https://twitter.com/settings/your_twitter_data) can be provided with `--archive`([more info in the docs](https://robertoszek.github.io/pleroma-bot/gettingstarted/usage/#using-an-archive)) +- Links to Twitter attachments (video, images) are no longer explicitly included on the post's body text by default. You can choose to keep adding them with `keep_media_links`. This option does *not* affect the upload of attachments. +- Youtube links can be replaced with `invidious` and `invidious_base_url` +## Enhancements +- `bio_text` is no longer a mandatory mapping on the config +- Hugely improved performance (around 4x) when processing tweets +- Implemented safety measures for avoiding collision with multiple instances of the bot running at the same time + ## [0.8.9] - 2021-12-05 ## Added - ```original_date``` and ```original_date_format``` for adding the original tweet's creation date to the post body diff --git a/FUNDING.yml b/FUNDING.yml new file mode 100644 index 0000000..2ba2307 --- /dev/null +++ b/FUNDING.yml @@ -0,0 +1,13 @@ +# These are supported funding model platforms + +github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2] +patreon: # Replace with a single Patreon username +open_collective: # Replace with a single Open Collective username +ko_fi: robertoszek +tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel +community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry +liberapay: robertoszek +issuehunt: # Replace with a single IssueHunt username +otechie: # Replace with a single Otechie username +lfx_crowdfunding: # Replace with a single LFX Crowdfunding project-name e.g., cloud-foundry +custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2'] diff --git a/LICENSE.md b/LICENSE.md index 6d4af3b..b0d2ea5 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -MIT License Copyright (c) 2021 Roberto Chamorro / project contributors +MIT License Copyright (c) 2022 Roberto Chamorro / project contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 3a2803a..de2ffff 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Stork (pleroma-bot) -[![Build Status](https://travis-ci.com/robertoszek/pleroma-bot.svg?branch=develop)](https://travis-ci.com/robertoszek/pleroma-bot) +[![Build Status](https://travis-ci.com/robertoszek/pleroma-bot.svg?branch=develop)](https://app.travis-ci.com/github/robertoszek/pleroma-bot) [![Version](https://img.shields.io/pypi/v/pleroma-bot.svg)](https://pypi.org/project/pleroma-bot/) [![AUR version](https://img.shields.io/aur/version/python-pleroma-bot)](https://aur.archlinux.org/packages/python-pleroma-bot) [![codecov](https://codecov.io/gh/robertoszek/pleroma-bot/branch/master/graph/badge.svg?token=0c4Gzv4HjC)](https://codecov.io/gh/robertoszek/pleroma-bot) @@ -9,7 +9,7 @@ ![Stork](img/stork-smaller.svg) -Mirror one or multiple Twitter accounts in Pleroma/Mastodon. +Mirror your favourite Twitter accounts in the Fediverse, so you can follow their updates from the comfort of your favorite instance. Or migrate your own to the Fediverse using a Twitter [archive](https://twitter.com/settings/your_twitter_data). [![Documentation](img/docs.png)](https://robertoszek.github.io/pleroma-bot) @@ -23,7 +23,7 @@ For precisely those cases I've written this Python project that automates them, ## Features So basically, it does the following: - +* Can parse a Twitter [archive](https://twitter.com/settings/your_twitter_data), moving all your tweets to the Fediverse * Retrieves **tweets** and posts them on the Fediverse account if their timestamp is newer than the last post. * Can filter out RTs or not * Can filter out replies or not @@ -53,7 +53,7 @@ Here's a list of the available packages. ## Usage ```console -$ pleroma-bot [--noProfile] [--forceDate [FORCEDATE]] [-c CONFIG] +$ pleroma-bot [-c CONFIG] [-l LOG] [--noProfile] [--forceDate [FORCEDATE]] [-a ARCHIVE] ``` ```console @@ -65,6 +65,9 @@ optional arguments: path of config file (config.yml) to use and parse. If not specified, it will try to find it in the current working directory. + -l LOG, --log LOG path of log file (error.log) to create. If not + specified, it will try to store it at your config file + path -n, --noProfile skips Fediverse profile update (no background image, profile image, bio text, etc.) --forceDate [FORCEDATE] @@ -73,6 +76,9 @@ optional arguments: supplied to only force it for that particular user in the config -s, --skipChecks skips first run checks + -a ARCHIVE, --archive ARCHIVE + path of the Twitter archive file (zip) to use for + posting tweets. --verbose, -v --version show program's version number and exit ``` @@ -87,107 +93,30 @@ If you plan on retrieving tweets from an account which has their tweets **protec * Access Token Key and Secret. You'll also find them on your project app keys and tokens section at [Twitter's Developer Portal](https://developer.twitter.com/en/portal/dashboard). Alternatively, you can obtain the Access Token and Secret by running [this](https://github.com/joestump/python-oauth2/wiki/Twitter-Three-legged-OAuth-Python-3.0) locally, while being logged in with a Twitter account which follows or is the owner of the protected account -Create a ```config.yml``` file in the same path where you are calling ```pleroma-bot```. There's a config example in this repo called ```config.yml.sample``` that can help you when filling yours out: +### Configuration + +Create a ```config.yml``` file in the same path where you are calling ```pleroma-bot``` (or use the `--config` argument to specify a different path). + +There's a config example in this repo called ```config.yml.sample``` that can help you when filling yours out. + +For more information you can refer to the ["Configuration" page](https://robertoszek.github.io/pleroma-bot/gettingstarted/configuration/) on the docs. + +Here's what a minimal config looks like: ```yaml -twitter_base_url: https://api.twitter.com/1.1 -# Change this to your Fediverse instance -pleroma_base_url: https://pleroma.robertoszek.xyz -# (optional) Change this to your preferred nitter instance -nitter_base_url: https://nitter.net +# Change this to your target Fediverse instance +pleroma_base_url: https://pleroma.instance # How many tweets to get in every execution # Twitter's API hard limit is 3,200 max_tweets: 40 # Twitter bearer token -twitter_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX -# List of users and their attributes +twitter_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX users: -- twitter_username: KyleBosman - pleroma_username: KyleBosman - # Mastodon/Pleroma token obtained by following the README.md +- twitter_username: User1 + pleroma_username: MyPleromaUser1 + # Mastodon/Pleroma bearer token pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - # (optional) keys and secrets for using OAuth 1.0a (for protected accounts) - consumer_key: xxxxxxxxxxxxxxxxxxxxxxxxx - consumer_secret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - access_token_key: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - access_token_secret: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx - # If you want to add a link to the original status or not - signature: true - # (optional) If you want to download Twitter attachments and add them to the Pleroma posts. - # By default they are not - media_upload: true - # (optional) If twitter links should be changed to nitter ones. By default they are not - nitter: true - # (optional) If mentions should be transformed to links to the mentioned Twitter profile - rich_text: true - # (optional) visibility of the post. Must one of the following: public, unlisted, private, direct - # by default is "unlisted" - visibility: "unlisted" - # (optional) Force all posts for this account to be sensitive or not - # The NSFW banner for the instance will be shown for attachments as a warning if true - # If not defined, the original tweet sensitivity will be used on a tweet by tweet basis - sensitive: false - # (optional) Include the creation date of the tweet on the Fediverse post body - original_date: true - # (optional) Date format to use when adding the creation date of the tweet to the Fediverse post - original_date_format: "%Y/%m/%d %H:%" - # (optional) If RTs are to be also be posted in the Fediverse account. By default they are included - include_rts: false - # (optional) If replies are to be also posted in the Fediverse account. By default they are included - include_replies: false - # (optional) List of hashtags to use for filtering out tweets which don't include any of them - hashtags: - - sponsored - # (optional) How big attachments can be before being ignored and not being uploaded to the Fediverse post - # Examples: "30MB", "1.5GB", "0.5TB" - file_max_size: 500MB - # additional custom-named attributes - support_account: robertoszek - # you can use any attribute from 'user' inside a string with {{ attr_name }} and it will be replaced - # with the attribute value. e.g. {{ support_account }} - bio_text: "\U0001F916 BEEP BOOP \U0001F916 \nI'm a bot that mirrors {{ twitter_username }} Twitter's\ - \ account. \nAny issues please contact @{{ support_account }} \n \n " # username will be replaced by its value - # Optional metadata fields and values for the Pleroma profile - fields: - - name: "\U0001F426 Birdsite" - value: "{{ twitter_url }}" - - name: "Status" - value: "I am completely operational, and all my circuits are functioning perfectly." - - name: "Source" - value: "https://gitea.robertoszek.xyz/robertoszek/pleroma-twitter-info-grabber" -# Mastodon instance example -- twitter_username: WoolieWoolz - pleroma_username: 24660 - pleroma_base_url: https://botsin.space - pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - # Mastodon doesn't support rich text! - rich_text: false - signature: true - nitter: true - visibility: "unlisted" - media_upload: true - max_tweets: 50 - bio_text: "\U0001F916 BEEP BOOP \U0001F916 \nI'm a bot that mirrors {{ twitter_username }} Twitter's\ - \ account. \nAny issues please contact @{{ support_account }} \n \n " # username will be replaced by its value - fields: - - name: "\U0001F426 Birdsite" - value: "{{ twitter_url }}" - - name: "Status" - value: "I am completely operational, and all my circuits are functioning perfectly." - - name: "Source" - value: "https://gitea.robertoszek.xyz/robertoszek/pleroma-twitter-info-grabber" -# Minimal config example -- twitter_username: arstechnica - pleroma_username: mynewsbot - pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - bio_text: "" ``` -Changing the ```users``` to the desired ones. You can add as many users as needed. - -Also change the following to your Pleroma/Mastodon instance URL: -```yaml -pleroma_base_url: https://pleroma.robertoszek.xyz -``` ### Running If you're running the bot for the first time it will ask you for the date you wish to start retrieving tweets from (it will gather all from that date up to the present). @@ -203,7 +132,7 @@ For example: $ pleroma-bot --forceDate WoolieWoolz ``` -If the ```--noProfile``` argument is passed, *only* new tweets will be posted. The profile picture, banner, display name and bio will **not** be updated on the Fediverse account. +If the --noProfile argument is passed, the profile picture, banner, display name and bio will **not** be updated on the Fediverse account. However, it will still gather and post the tweets following your config's parameters. NOTE: An ```error.log``` file will be created at the path from which ```pleroma-bot``` is being called. @@ -243,7 +172,7 @@ Patches, pull requests, and bug reports are more than [welcome](https://github.c MIT License -Copyright (c) 2021 Roberto Chamorro / project contributors +Copyright (c) 2022 Roberto Chamorro / project contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -264,6 +193,6 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. **Tested and confirmed working against** : -* ```Pleroma BE 2.0.50-2547-g5c2b6922-develop``` +* ```Pleroma BE 2.2.50-711-g100e34b4-develop``` * ```Mastodon v3.2.1``` * ```Twitter API v1.1 and v2``` diff --git a/config-minimal.yml.sample b/config-minimal.yml.sample new file mode 100644 index 0000000..e14a694 --- /dev/null +++ b/config-minimal.yml.sample @@ -0,0 +1,12 @@ +# Change this to your Fediverse instance +pleroma_base_url: https://pleroma.instance +# How many tweets to get in every execution +# Twitter's API hard limit is 3,200 +max_tweets: 40 +# Twitter bearer token +twitter_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +users: +- twitter_username: User1 + pleroma_username: MyPleromaUser1 + # Mastodon/Pleroma bearer token + pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX \ No newline at end of file diff --git a/config.yml.sample b/config.yml.sample index c5ef3ac..4f7ce58 100644 --- a/config.yml.sample +++ b/config.yml.sample @@ -50,6 +50,7 @@ users: support_account: robertoszek # you can use any attribute from 'user' inside a string with {{ attr_name }} and it will be replaced # with the attribute value. e.g. {{ support_account }} + # (optional) Text to be appended to the Twitter account bio text bio_text: "\U0001F916 BEEP BOOP \U0001F916 \nI'm a bot that mirrors {{ twitter_username }} Twitter's\ \ account. \nAny issues please contact @{{ support_account }} \n \n " # username will be replaced by its value # Optional metadata fields and values for the Pleroma profile diff --git a/docs/contribute/contributing.md b/docs/contribute/contributing.md index db0eb4d..64026b4 100644 --- a/docs/contribute/contributing.md +++ b/docs/contribute/contributing.md @@ -1,5 +1,8 @@ # Contributing +## Funding +[![Donate using Liberapay](https://liberapay.com/assets/widgets/donate.svg)](https://liberapay.com/robertoszek/donate) [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/robertoszek) + ## Code Patches, pull requests, and bug reports are more than [welcome](https://github.com/robertoszek/pleroma-bot/issues/new/choose), please keep the style consistent with the original source. diff --git a/docs/gettingstarted/beforerunning.md b/docs/gettingstarted/beforerunning.md index 3cd680d..38d3360 100644 --- a/docs/gettingstarted/beforerunning.md +++ b/docs/gettingstarted/beforerunning.md @@ -4,9 +4,15 @@ If you haven't already, you need to [apply for a Twitter developer account](http The process involves some review of the developer account application by Twitter and it's very likely you'll be asked for some details pertaining your usecase. It usually doesn't take longer than a day or two to complete the application, the back and forth is mostly automated on their part. +Additionally, Twitter introduced a new tier of access (Elevated) to their API projects and although existing projects (before Nov 2020) were promoted automatically, new users will only get Essential access instead by default, in which requests to API v1.1 are disabled. + +We still use v1.1 for downloading videos and profile banners, and as of now there is no available alternative in v2, so you'll need Elevated access for the bot to function properly until further notice. + +You can apply for Elevated access [here](https://developer.twitter.com/en/portal/products/elevated). + ## Twitter tokens -Once you have a Twitter developer account, you need to access your [dashboard](https://developer.twitter.com/en/portal/dashboard) and create a new project (so your app has V2 access) and also create a new app associated to that new project. +Once you have a Twitter developer account, you need to access your [dashboard](https://developer.twitter.com/en/portal/dashboard) and create a new project (so your app has v2 access) and also create a new app associated to that new project. Now, enter your new application "Keys and tokens" section, copy and safely store all of your tokens. diff --git a/docs/gettingstarted/configuration.md b/docs/gettingstarted/configuration.md index 4f68af6..e681287 100644 --- a/docs/gettingstarted/configuration.md +++ b/docs/gettingstarted/configuration.md @@ -59,7 +59,6 @@ users: - twitter_username: User1 pleroma_username: MyPleromaUser1 pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX - bio_text: "" ``` ## Mappings @@ -67,43 +66,44 @@ users: Every mapping that ```pleroma-bot``` understands is listed below with a description, which allows you to further customize how each user should behave. -| Mapping | Optional | Default | Description | -|:--------------------|:----------:|:----------------------------|:----------------------------------------------------------------------| -| twitter_base_url | No | | Twitter API fallback URL. It should be 'https://api.twitter.com/1.1' | -| pleroma_base_url | No | | Your Fediverse instance URL | -| max_tweets | No | | How many tweets to get in every execution (Twitter's API hard limit is 3,200)| -| twitter_token | No | | Twitter bearer token used for authentication | -| consumer_key | Yes | | OAuth 1.0a Twitter Consumer Key (only needed for protected accounts) | -| consumer_secret | Yes | | OAuth 1.0a Twitter Consumer Secret (only needed for protected accounts) | -| access_token_key | Yes | | OAuth 1.0a Twitter Access Token Key (only needed for protected accounts) | -| access_token_secret | Yes | | OAuth 1.0a Twitter Access Token Secret (only needed for protected accounts) | -| nitter | Yes | false | If Twitter links should be changed to nitter ones | -| nitter_base_url | Yes | https://nitter.net | Change this to your preferred nitter instance | -| signature | Yes | false | Add a link to the original status | -| media_upload | Yes | false | Download Twitter attachments and add them to the Fediverse posts | -| rich_text | Yes | false | Transform mentions to links pointing to the mentioned Twitter profile | -| include_rts | Yes | false | Include RTs when posting tweets in the Fediverse account | -| include_replies | Yes | false | Include replies when posting tweets in the Fediverse account | -| hashtags | Yes | | List of hashtags to filter out tweets which don't include any of them | -| visibility | Yes | unlisted | Visibility of the post. Must one of the following: public, unlisted, private, direct | -| sensitive | Yes | original tweet sensitivity | Force all posts to be sensitive (NSFW) or not | -| file_max_size | Yes | | How big attachments can be before being ignored. Examples: "30MB", "1.5GB", "0.5TB" | -| delay_post | Yes | 0.5 | How long to wait (in seconds) between submitting posts to the Fedi instance (useful when trying to avoid rate limits)| -| tweet_ids | Yes | | List of specific tweet IDs to retrieve and post | -| twitter_bio | Yes | true | Append Twitter's bio to Pleroma/Mastodon target user | -| original_date | Yes | false | Include the creation date of the tweet on the Fediverse post body | -| original_date_format| Yes | "%Y-%m-%d %H:%M" | Date format to use when adding the creation date of the tweet to the Fediverse post | - - - -There a few mapping *exclusive* to users: +| Mapping | Optional | Default | Description | +|:---------------------|:----------:|:---------------------------|:----------------------------------------------------------------------------------------------------------------------| +| twitter_base_url | No | | Twitter API fallback URL. It should be 'https://api.twitter.com/1.1' | +| pleroma_base_url | No | | Your Fediverse instance URL | +| max_tweets | No | | How many tweets to get in every execution (Twitter's API hard limit is 3,200) | +| twitter_token | No | | Twitter bearer token used for authentication | +| consumer_key | Yes | | OAuth 1.0a Twitter Consumer Key (only needed for protected accounts) | +| consumer_secret | Yes | | OAuth 1.0a Twitter Consumer Secret (only needed for protected accounts) | +| access_token_key | Yes | | OAuth 1.0a Twitter Access Token Key (only needed for protected accounts) | +| access_token_secret | Yes | | OAuth 1.0a Twitter Access Token Secret (only needed for protected accounts) | +| nitter | Yes | false | If Twitter links should be changed to nitter ones | +| nitter_base_url | Yes | https://nitter.net | Change this to your preferred nitter instance | +| signature | Yes | false | Add a link to the original status | +| media_upload | Yes | false | Download Twitter attachments and add them to the Fediverse posts | +| rich_text | Yes | false | Transform mentions to links pointing to the mentioned Twitter profile | +| include_rts | Yes | false | Include RTs when posting tweets in the Fediverse account | +| include_replies | Yes | false | Include replies when posting tweets in the Fediverse account | +| hashtags | Yes | | List of hashtags to use to filter out tweets which don't include any of them | +| visibility | Yes | unlisted | Visibility of the post. Must one of the following: public, unlisted, private, direct | +| sensitive | Yes | original tweet sensitivity | Force all posts to be sensitive (NSFW) or not | +| file_max_size | Yes | | How big attachments can be before being ignored. Examples: "30MB", "1.5GB", "0.5TB" | +| delay_post | Yes | 0.5 | How long to wait (in seconds) between submitting posts to the Fedi instance (useful when trying to avoid rate limits) | +| tweet_ids | Yes | | List of specific tweet IDs to retrieve and post | +| twitter_bio | Yes | true | Append Twitter's bio to Pleroma/Mastodon target user | +| original_date | Yes | false | Include the creation date of the tweet on the Fediverse post body | +| original_date_format | Yes | "%Y-%m-%d %H:%M" | Date format to use when adding the creation date of the tweet to the Fediverse post | +| keep_media_links | Yes | false | Keep redundant media links on the tweet text or not (`https://twitter.com//status//photo/1`) | +| invidious | Yes | false | If Youtube links should be replaced with invidious ones | +| invidious_base_url | Yes | https://yewtu.be | Change this to your preferred invidious instance | + +There a few mappings *exclusive* to users: | User mapping | Optional | Default | Description | |:-----------------|:----------:|:----------------------------|:----------------------------------------------------------------------| -| twitter_username | No | | Username of Twitter account to mirror | +| twitter_username | No | | Username of Twitter account to mirror (can be a list) | | pleroma_username | No | | Username of target Fediverse account to post content and update profile | | pleroma_token | No | | Bearer token of target Fediverse account | -| bio_text | No | | Text to be appended to the Twitter account bio text | +| bio_text | Yes | | Text to be appended to the Twitter account bio text | | fields | Yes | | Optional metadata fields (sequence of name-value pairs) for the Fediverse profile | diff --git a/docs/gettingstarted/installation.md b/docs/gettingstarted/installation.md index 7bf1384..a06c7c3 100644 --- a/docs/gettingstarted/installation.md +++ b/docs/gettingstarted/installation.md @@ -37,12 +37,12 @@ First you need to intall ```pleroma-bot``` on your system. There are multiple me Either way, here's a list of the dependencies in case you need them: -| Name | Git repo | Docs -| --------------- | ---------------------------------------------- | ----------------------------------------------------------- | -| python-oauthlib | [GitHub](https://github.com/oauthlib/oauthlib) | [Documentation](https://oauthlib.readthedocs.io/en/latest/) | -| python-pyaml | [GitHub](https://github.com/yaml/pyyaml) | [Documentation](https://pyyaml.org/wiki/PyYAMLDocumentation) | -| python-requests | [GitHub](https://github.com/psf/requests) | [Documentation](https://requests.readthedocs.io/) | -| python-requests-oauthlib | [GitHub](https://github.com/requests/requests-oauthlib) | [Documentation](https://requests-oauthlib.readthedocs.org) | +| Name | Git repo | Docs | +|--------------------------|---------------------------------------------------------|--------------------------------------------------------------| +| python-oauthlib | [GitHub](https://github.com/oauthlib/oauthlib) | [Documentation](https://oauthlib.readthedocs.io/en/latest/) | +| python-pyaml | [GitHub](https://github.com/yaml/pyyaml) | [Documentation](https://pyyaml.org/wiki/PyYAMLDocumentation) | +| python-requests | [GitHub](https://github.com/psf/requests) | [Documentation](https://requests.readthedocs.io/) | +| python-requests-oauthlib | [GitHub](https://github.com/requests/requests-oauthlib) | [Documentation](https://requests-oauthlib.readthedocs.org) | ## Test the installation @@ -79,11 +79,11 @@ Once installed using your preferred method, test that the package has been corre ff= `==:::""",,,,________ - usage: cli.py [-h] [-c CONFIG] [-l LOG] [-n] [--forceDate [FORCEDATE]] [-s] - [--verbose] [--version] + usage: pleroma-bot [-h] [-c CONFIG] [-l LOG] [-n] [--forceDate [FORCEDATE]] [-s] + [-a ARCHIVE] [--verbose] [--version] Bot for mirroring one or multiple Twitter accounts in Pleroma/Mastodon. - + optional arguments: -h, --help show this help message and exit -c CONFIG, --config CONFIG @@ -101,6 +101,9 @@ Once installed using your preferred method, test that the package has been corre supplied to only force it for that particular user in the config -s, --skipChecks skips first run checks + -a ARCHIVE, --archive ARCHIVE + path of the Twitter archive file (zip) to use for + posting tweets. --verbose, -v --version show program's version number and exit ``` @@ -134,11 +137,11 @@ Once installed using your preferred method, test that the package has been corre ff= `==:::""",,,,________ - usage: cli.py [-h] [-c CONFIG] [-l LOG] [-n] [--forceDate [FORCEDATE]] [-s] - [--verbose] [--version] + usage: pleroma-bot [-h] [-c CONFIG] [-l LOG] [-n] [--forceDate [FORCEDATE]] [-s] + [-a ARCHIVE] [--verbose] [--version] Bot for mirroring one or multiple Twitter accounts in Pleroma/Mastodon. - + optional arguments: -h, --help show this help message and exit -c CONFIG, --config CONFIG @@ -156,6 +159,9 @@ Once installed using your preferred method, test that the package has been corre supplied to only force it for that particular user in the config -s, --skipChecks skips first run checks + -a ARCHIVE, --archive ARCHIVE + path of the Twitter archive file (zip) to use for + posting tweets. --verbose, -v --version show program's version number and exit ``` @@ -190,10 +196,10 @@ Once installed using your preferred method, test that the package has been corre usage: cli.py [-h] [-c CONFIG] [-l LOG] [-n] [--forceDate [FORCEDATE]] [-s] - [--verbose] [--version] + [-a ARCHIVE] [--verbose] [--version] Bot for mirroring one or multiple Twitter accounts in Pleroma/Mastodon. - + optional arguments: -h, --help show this help message and exit -c CONFIG, --config CONFIG @@ -211,6 +217,9 @@ Once installed using your preferred method, test that the package has been corre supplied to only force it for that particular user in the config -s, --skipChecks skips first run checks + -a ARCHIVE, --archive ARCHIVE + path of the Twitter archive file (zip) to use for + posting tweets. --verbose, -v --version show program's version number and exit ``` diff --git a/docs/gettingstarted/usage.md b/docs/gettingstarted/usage.md index c4b3c5a..fc97942 100644 --- a/docs/gettingstarted/usage.md +++ b/docs/gettingstarted/usage.md @@ -59,4 +59,16 @@ Arguments ```--config``` and ```--log``` can be used to specify a specific path $ pleroma-bot --config /path/to/config.yml --log /path/to/error.log ``` -When these arguments are omitted, ```config.yml``` from the current directory will be used as a configuration file and an ```error.log``` file will be written to the current working directory. \ No newline at end of file +When these arguments are omitted, ```config.yml``` from the current directory will be used as a configuration file and an ```error.log``` file will be written to the current working directory. + +## Using an archive + +A Twitter [archive](https://twitter.com/settings/your_twitter_data) can also be provided with `--archive`: + +```console +$ pleroma-bot --archive /path/to/archive.zip +``` + +This is particularly useful when trying to circumvent Twitter's API limitations, when you need to copy more than 3200 tweets or from an earlier date than `2010-11-06T00:00:00Z`. + +It can also be used as a way of transferring all of your Twitter's account data to a Fediverse instance and making the migration process easier. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index 17b9850..5658577 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,19 +1,21 @@ # Stork (pleroma-bot) + ![Stork](/pleroma-bot/images/logo.png) -Mirror your favourite Twitter accounts in the Fediverse, so you can follow their updates from the comfort of your own instance. +Mirror your favourite Twitter accounts in the Fediverse, so you can follow their updates from the comfort of your favorite instance. Or migrate your own to the Fediverse using a Twitter [archive](https://twitter.com/settings/your_twitter_data). -[![Build Status](https://travis-ci.com/robertoszek/pleroma-bot.svg?branch=master)](https://travis-ci.com/robertoszek/pleroma-bot) +[![Build Status](https://travis-ci.com/robertoszek/pleroma-bot.svg?branch=master)](https://app.travis-ci.com/github/robertoszek/pleroma-bot) [![Version](https://img.shields.io/pypi/v/pleroma-bot.svg)](https://pypi.org/project/pleroma-bot/) [![codecov](https://codecov.io/gh/robertoszek/pleroma-bot/branch/master/graph/badge.svg?token=0c4Gzv4HjC)](https://codecov.io/gh/robertoszek/pleroma-bot) [![Python 3.6](https://img.shields.io/badge/python-3.6+-blue.svg)](https://www.python.org/downloads/release/python-360/) -[![Requires.io (branch)](https://img.shields.io/requires/github/robertoszek/pleroma-bot/master)](https://requires.io/github/robertoszek/pleroma-bot/requirements/?branch=master) [![License](https://img.shields.io/github/license/robertoszek/pleroma-bot)](https://github.com/robertoszek/pleroma-bot/blob/master/LICENSE.md) + [:material-file-document-multiple-outline: Get started](/pleroma-bot/gettingstarted/installation/){: .md-button } [:material-download-outline: Download](https://github.com/robertoszek/pleroma-bot/releases/latest){: .md-button } [:material-arch: AUR (Arch)](https://aur.archlinux.org/packages/python-pleroma-bot){: .md-button } ## Features +* [x] Can parse a Twitter [archive](https://twitter.com/settings/your_twitter_data), moving all your tweets to the Fediverse * [x] Retrieves **tweets** and posts them on the Fediverse account * [x] Can filter out RTs * [x] Can filter out replies @@ -29,4 +31,8 @@ Mirror your favourite Twitter accounts in the Fediverse, so you can follow their * [x] *Bio text* * [x] Customize Fediverse account's **metadata fields** (e.g. point to the original Twitter account) +## Funding +If you feel like supporting the creator or buying him a beer, you can donate through Ko-fi or Liberapay + +[![Donate using Liberapay](https://liberapay.com/assets/widgets/donate.svg)](https://liberapay.com/robertoszek/donate) [![ko-fi](https://ko-fi.com/img/githubbutton_sm.svg)](https://ko-fi.com/robertoszek) diff --git a/pleroma_bot/__init__.py b/pleroma_bot/__init__.py index b512166..37f639d 100644 --- a/pleroma_bot/__init__.py +++ b/pleroma_bot/__init__.py @@ -3,7 +3,7 @@ import locale import logging -__version__ = "0.8.9" +__version__ = "1.0.0" class StdOutFilter(logging.Filter): @@ -31,7 +31,7 @@ class CustomFormatter(logging.Formatter): logging.ERROR: red + "โœ– " + format_l + reset, logging.CRITICAL: bold_red + format_l + reset, } - else: + else: # pragma: win32 cover FORMATS = { logging.DEBUG: format_r, logging.INFO: "ยก " + format_r, @@ -95,7 +95,7 @@ def format(self, record): # fill env locale vars in case we're running in other platforms default_lang, default_enc = locale.getdefaultlocale() -if "LANG" not in os.environ: +if "LANG" not in os.environ: # pragma: win32 cover os.environ["LANG"] = default_lang os.environ["LANGUAGE"] = f"{default_lang}.{default_enc}" diff --git a/pleroma_bot/_error.py b/pleroma_bot/_error.py new file mode 100644 index 0000000..6bbdd12 --- /dev/null +++ b/pleroma_bot/_error.py @@ -0,0 +1,15 @@ +from .i18n import _ + + +class TimeoutLocker(TimeoutError): + """Raised when the lock could not be acquired in *timeout* seconds.""" + + def __init__(self, lock_file: str) -> None: + #: The path of the file lock. + self.lock_file = lock_file + + def __str__(self) -> str: + return _( + "The file lock '{}' could not be acquired. Is another instance " + "of pleroma-bot running?" + ).format(self.lock_file) diff --git a/pleroma_bot/_pin.py b/pleroma_bot/_pin.py index 2637f38..f28d0d8 100644 --- a/pleroma_bot/_pin.py +++ b/pleroma_bot/_pin.py @@ -17,7 +17,10 @@ def pin_pleroma(self, id_post): :returns: ID of post pinned :rtype: str """ - pinned_file = os.path.join(self.user_path, "pinned_id_pleroma.txt") + # Only check pinned for 1 user + t_user = self.twitter_username[0] + + pinned_file = os.path.join(self.user_path[t_user], "pinned_id_pleroma.txt") self.unpin_pleroma(pinned_file) pin_url = f"{self.pleroma_base_url}/api/v1/statuses/{id_post}/pin" @@ -37,7 +40,10 @@ def unpin_pleroma(self, pinned_file): :param pinned_file: path to file containing post ID """ - pinned_file_twitter = os.path.join(self.user_path, "pinned_id.txt") + # Only check pinned for 1 user + t_user = self.twitter_username[0] + + pinned_file_twitter = os.path.join(self.user_path[t_user], "pinned_id.txt") previous_pinned_post_id = None if os.path.isfile(pinned_file): with open(os.path.join(pinned_file), "r") as file: @@ -111,9 +117,12 @@ def _get_pinned_tweet_id(self): :returns: ID of currently pinned tweet """ + # Only get pin for 1 user + t_user = self.twitter_username[0] + url = ( f"{self.twitter_base_url_v2}/users/" - f"by/username/{self.twitter_username}" + f"by/username/{t_user}" ) params = { "user.fields": "pinned_tweet_id", diff --git a/pleroma_bot/_pleroma.py b/pleroma_bot/_pleroma.py index cd2f56e..cb4be30 100644 --- a/pleroma_bot/_pleroma.py +++ b/pleroma_bot/_pleroma.py @@ -67,69 +67,64 @@ def post_pleroma(self, tweet: tuple, poll: dict, sensitive: bool) -> str: :returns: id of post :rtype: str """ - # TODO: transform twitter links to nitter links, if self.nitter - # 'true' in resolved shortened urls + pleroma_post_url = f"{self.pleroma_base_url}/api/v1/statuses" pleroma_media_url = f"{self.pleroma_base_url}/api/v1/media" - tweet_id = tweet[0] - tweet_text = tweet[1] - tweet_date = tweet[2] + tweet_id, tweet_text, tweet_date = tweet tweet_folder = os.path.join(self.tweets_temp_path, tweet_id) - media_files = os.listdir(tweet_folder) + media_ids = [] if self.media_upload: - for file in media_files: - media_file = open(os.path.join(tweet_folder, file), "rb") - file_size = os.stat(os.path.join(tweet_folder, file)).st_size - mime_type = guess_type(os.path.join(tweet_folder, file)) - timestamp = str(datetime.now().timestamp()) - file_name = ( - f"pleromapyupload_" - f"{timestamp}" - f"_" - f"{random_string(10)}" - f"{mimetypes.guess_extension(mime_type)}" - ) - file_description = (file_name, media_file, mime_type) - files = {"file": file_description} - response = requests.post( - pleroma_media_url, headers=self.header_pleroma, files=files - ) - try: - if not response.ok: - response.raise_for_status() - except requests.exceptions.HTTPError: - if response.status_code == 413: - size_msg = _( - "Exception occurred" - "\nMedia size too large:" - "\nFilename: {file}" - "\nSize: {size}MB" - "\nConsider increasing the attachment" - "\n size limit of your instance" - ).format(file=file, size=round(file_size / 1048576, 2)) - logger.error(size_msg) - pass - else: - response.raise_for_status() - try: - media_ids.append(json.loads(response.text)["id"]) - except (KeyError, JSONDecodeError): - logger.warning( - _("Error uploading media:\t{}").format(str(response.text)) + if os.path.isdir(tweet_folder): + media_files = os.listdir(tweet_folder) + for file in media_files: + file_path = os.path.join(tweet_folder, file) + media_file = open(file_path, "rb") + file_size = os.stat(os.path.join(tweet_folder, file)).st_size + size_mb = round(file_size / 1048576, 2) + + mime_type = guess_type(os.path.join(tweet_folder, file)) + timestamp = str(datetime.now().timestamp()) + file_name = ( + f"pleromapyupload_" + f"{timestamp}" + f"_" + f"{random_string(10)}" + f"{mimetypes.guess_extension(mime_type)}" ) - pass + file_description = (file_name, media_file, mime_type) + files = {"file": file_description} + response = requests.post( + pleroma_media_url, headers=self.header_pleroma, files=files + ) + try: + if not response.ok: + response.raise_for_status() + except requests.exceptions.HTTPError: + if response.status_code == 413: + size_msg = _( + "Exception occurred" + "\nMedia size too large:" + "\nFilename: {file}" + "\nSize: {size}MB" + "\nConsider increasing the attachment" + "\n size limit of your instance" + ).format(file=file_path, size=size_mb) + logger.error(size_msg) + pass + else: + response.raise_for_status() + try: + media_ids.append(json.loads(response.text)["id"]) + except (KeyError, JSONDecodeError): + logger.warning( + _("Error uploading media:\t{}").format( + str(response.text) + ) + ) + pass - if self.signature: - signature = f"\n\n ๐Ÿฆ๐Ÿ”—: {self.twitter_url}/status/{tweet_id}" - tweet_text = f"{tweet_text} {signature}" - if self.original_date: - date = datetime.strftime( - datetime.strptime(tweet_date, "%Y-%m-%dT%H:%M:%S.000Z"), - self.original_date_format, - ) - tweet_text = f"{tweet_text} \n\n[{date}]" # config setting override tweet attr if self.sensitive: sensitive = self.sensitive @@ -175,23 +170,28 @@ def update_pleroma(self): :returns: None """ + # Only update 1 user + t_user = self.twitter_username[0] # Get the biggest resolution for the profile picture (400x400) # instead of 'normal' - if self.profile_image_url: - profile_img_big = re.sub(r"normal", "400x400", self.profile_image_url) + if self.profile_image_url[t_user]: + profile_img_big = re.sub( + r"normal", "400x400", + self.profile_image_url[t_user] + ) response = requests.get(profile_img_big, stream=True) if not response.ok: response.raise_for_status() response.raw.decode_content = True - with open(self.avatar_path, "wb") as outfile: + with open(self.avatar_path[t_user], "wb") as outfile: shutil.copyfileobj(response.raw, outfile) - if self.profile_banner_url: - response = requests.get(self.profile_banner_url, stream=True) + if self.profile_banner_url[t_user]: + response = requests.get(self.profile_banner_url[t_user], stream=True) if not response.ok: response.raise_for_status() response.raw.decode_content = True - with open(self.header_path, "wb") as outfile: + with open(self.header_path[t_user], "wb") as outfile: shutil.copyfileobj(response.raw, outfile) # Set it on Pleroma @@ -202,13 +202,16 @@ def update_pleroma(self): for field_item in self.fields: field = (field_item["name"], field_item["value"]) fields.append(field) - data = {"note": self.bio_text, "display_name": self.display_name} + data = { + "note": self.bio_text[t_user], + "display_name": self.display_name[t_user] + } if self.profile_image_url: - data.update({"avatar": self.avatar_path}) + data.update({"avatar": self.avatar_path[t_user]}) if self.profile_banner_url: - data.update({"header": self.header_path}) + data.update({"header": self.header_path[t_user]}) if len(fields) > 4: raise Exception( @@ -223,18 +226,18 @@ def update_pleroma(self): files = {} timestamp = str(datetime.now().timestamp()) - if self.profile_image_url: - avatar = open(self.avatar_path, "rb") - avatar_mime_type = guess_type(self.avatar_path) + if self.profile_image_url[t_user]: + avatar = open(self.avatar_path[t_user], "rb") + avatar_mime_type = guess_type(self.avatar_path[t_user]) avatar_file_name = ( f"pleromapyupload_{timestamp}_" f"{random_string(10)}" f"{mimetypes.guess_extension(avatar_mime_type)}" ) files.update({"avatar": (avatar_file_name, avatar, avatar_mime_type)}) - if self.profile_banner_url: - header = open(self.header_path, "rb") - header_mime_type = guess_type(self.header_path) + if self.profile_banner_url[t_user]: + header = open(self.header_path[t_user], "rb") + header_mime_type = guess_type(self.header_path[t_user]) header_file_name = ( f"pleromapyupload_{timestamp}_" f"{random_string(10)}" diff --git a/pleroma_bot/_processing.py b/pleroma_bot/_processing.py index d332550..95306d4 100644 --- a/pleroma_bot/_processing.py +++ b/pleroma_bot/_processing.py @@ -5,8 +5,8 @@ import shutil import requests import mimetypes +from datetime import datetime -from ._utils import spinner # Try to import libmagic # if it fails just use mimetypes @@ -19,7 +19,6 @@ from .i18n import _ -@spinner(_("Processing tweets... ")) def process_tweets(self, tweets_to_post): """Transforms tweets for posting them to Pleroma Expands shortened URLs @@ -30,6 +29,8 @@ def process_tweets(self, tweets_to_post): :returns: Tweets ready to be published :rtype: list """ + # TODO: Break into smaller functions + # Remove RTs if include_rts is false if not self.include_rts: for tweet in tweets_to_post["data"][:]: @@ -50,6 +51,7 @@ def process_tweets(self, tweets_to_post): break except KeyError: pass + if self.hashtags: for tweet in tweets_to_post["data"][:]: try: @@ -67,30 +69,110 @@ def process_tweets(self, tweets_to_post): for tweet in tweets_to_post["data"]: media = [] + logger.debug(tweet["id"]) tweet["text"] = _expand_urls(self, tweet) tweet["text"] = html.unescape(tweet["text"]) + + # Download media only if we plan to upload it later + if self.media_upload: + try: + if self.archive: + for item in tweet["extended_entities"]["media"]: + if item["type"] == "photo": + item["url"] = item["media_url"] + media.append(item) + else: + m_k = False + att = "attachments" in tweet.keys() + if att: + m_k = "media_keys" in tweet["attachments"].keys() + if m_k and att: + includes_media = tweets_to_post["includes"]["media"] + for item in tweet["attachments"]["media_keys"]: + for media_include in includes_media: + media_url = _get_media_url( + self, item, media_include, tweet + ) + if media_url: + media.extend(media_url) + # Get RT tweet media + if "referenced_tweets" in tweet.keys(): # pragma: no cover + tweet_rt = {"data": tweet} + tw_data = tweet_rt["data"] + i = 0 + while "referenced_tweets" in tw_data.keys(): + for reference in tw_data["referenced_tweets"]: + retweeted = reference["type"] == "retweeted" + quoted = reference["type"] == "quoted" + if retweeted or quoted: + tweet_id = reference["id"] + tweet_rt = self._get_tweets("v2", tweet_id) + tw_data = tweet_rt["data"] + att = "attachments" in tw_data.keys() + if att: + attachments = tw_data["attachments"] + in_md = tweet_rt["includes"]["media"] + md_keys = attachments["media_keys"] + for item in md_keys: + for media_include in in_md: + media_url = _get_media_url( + self, + item, + media_include, + tweet_rt + ) + if media_url: + media.extend(media_url) + else: + break + i += 1 + if i > 3: + logger.debug( + _("Giving up, reference is too deep") + ) + break + except KeyError: + pass + if len(media) > 0: + # Create folder to store attachments related to the tweet ID + tweet_path = os.path.join(self.tweets_temp_path, tweet["id"]) + os.makedirs(tweet_path, exist_ok=True) + _download_media(self, media, tweet) + + if not self.keep_media_links: + tweet["text"] = _remove_media_links(self, tweet) if hasattr(self, "rich_text"): if self.rich_text: tweet["text"] = _replace_mentions(self, tweet) if self.nitter: - tweet["text"] = _replace_nitter(self, tweet) - - try: - for item in tweet["attachments"]["media_keys"]: - for media_include in tweets_to_post["includes"]["media"]: - media_url = _get_media_url( - self, item, media_include, tweet - ) - if media_url: - media.extend(media_url) - except KeyError: - pass - # Create folder to store attachments related to the tweet ID - tweet_path = os.path.join(self.tweets_temp_path, tweet["id"]) - os.makedirs(tweet_path, exist_ok=True) - # Download media only if we plan to upload it later - if self.media_upload: - _download_media(self, media, tweet) + tweet["text"] = _replace_url( + self, + tweet["text"], + "https://twitter.com", + self.nitter_base_url + ) + if self.invidious: + tweet["text"] = _replace_url( + self, + tweet["text"], + "https://youtube.com", + self.invidious_base_url + ) + if self.signature: + if self.archive: + t_user = self.twitter_ids[list(self.twitter_ids.keys())[0]] + else: + t_user = self.twitter_ids[tweet["author_id"]] + twitter_url = self.twitter_url[t_user] + signature = f"\n\n ๐Ÿฆ๐Ÿ”—: {twitter_url}/status/{tweet['id']}" + tweet["text"] = f"{tweet['text']} {signature}" + if self.original_date: + tweet_date = tweet["created_at"] + date = datetime.strftime( + datetime.strptime(tweet_date, "%Y-%m-%dT%H:%M:%S.000Z"), + self.original_date_format, + ) + tweet["text"] = f"{tweet['text']} \n\n[{date}]" # Process poll if exists and no media is used tweet["polls"] = _process_polls(self, tweet, media) @@ -111,7 +193,10 @@ def _process_polls(self, tweet, media): } response = requests.get( - poll_url, headers=self.header_twitter, params=params + poll_url, + headers=self.header_twitter, + params=params, + auth=self.auth ) if not response.ok: response.raise_for_status() @@ -204,25 +289,38 @@ def parse_size(size): return int(float(number) * units[unit]) -def _replace_nitter(self, tweet): - matching_pattern = "https://twitter.com" - matches = re.findall(matching_pattern, tweet["text"]) +def _replace_url(self, data, url, new_url): + matches = re.findall(url, data) for match in matches: - tweet["text"] = re.sub(match, self.nitter_base_url, tweet["text"]) + data = re.sub(match, new_url, data) + return data + + +def _remove_media_links(self, tweet): + regex = r"\bhttps?:\/\/twitter.com\/+[^\/:]+\/.*?(photo|video)\/\d*\b" + match = re.search(regex, tweet["text"]) + if match: + tweet["text"] = re.sub(match.group(), '', tweet["text"]) return tweet["text"] def _replace_mentions(self, tweet): matches = re.findall(r"\B\@\w+", tweet["text"]) for match in matches: + # TODO: Use nitter if asked (self.nitter) mention_link = f"[{match}](https://twitter.com/{match[1:]})" tweet["text"] = re.sub(match, mention_link, tweet["text"]) return tweet["text"] def _expand_urls(self, tweet): + # TODO: transform twitter links to nitter links, if self.nitter + # 'true' in resolved shortened urls + # Replace shortened links try: + if len(tweet["entities"]["urls"]) == 0: + raise KeyError for url_entity in tweet["entities"]["urls"]: matching_pattern = url_entity["url"] matches = re.findall(matching_pattern, tweet["text"]) @@ -243,22 +341,32 @@ def _expand_urls(self, tweet): group = match.group() # don't be brave trying to unwound an URL when it gets # cut off - if not group.__contains__("โ€ฆ"): + if ( + not group.__contains__("โ€ฆ") and not + group.startswith(self.nitter_base_url) + ): if not group.startswith(("http://", "https://")): group = f"http://{group}" - session = requests.Session() # so connections are - # recycled + # so connections are recycled + session = requests.Session() response = session.head(group, allow_redirects=True) if not response.ok: + logger.debug( + _( + "Couldn't expand the {}: {}" + ).format(response.url, response.status_code) + ) response.raise_for_status() - expanded_url = response.url - tweet["text"] = re.sub( - match.group(), expanded_url, tweet["text"] - ) + else: + expanded_url = response.url + tweet["text"] = re.sub( + group, expanded_url, tweet["text"] + ) return tweet["text"] def _get_media_url(self, item, media_include, tweet): + # TODO: Verify if video download is available on v2 and migrate to it media_urls = [] if item == media_include["media_key"]: # Video download not implemented in v2 yet @@ -279,9 +387,15 @@ def _get_media_url(self, item, media_include, tweet): def _get_best_bitrate_video(self, item): bitrate = 0 + url = "" for variant in item["video_info"]["variants"]: try: - if variant["bitrate"] >= bitrate: - return variant["url"] - except KeyError: + if "bitrate" in variant: + if int(variant["bitrate"]) >= bitrate: + bitrate = int(variant["bitrate"]) + url = variant["url"] + else: + continue + except KeyError: # pragma: no cover pass + return url diff --git a/pleroma_bot/_twitter.py b/pleroma_bot/_twitter.py index 9f53200..5412f3b 100644 --- a/pleroma_bot/_twitter.py +++ b/pleroma_bot/_twitter.py @@ -18,34 +18,68 @@ def _get_twitter_info(self): :return: None """ - twitter_user_url = ( - f"{self.twitter_base_url}" - f"/users/show.json?screen_name=" - f"{self.twitter_username}" - ) - response = requests.get( - twitter_user_url, headers=self.header_twitter, auth=self.auth - ) - if not response.ok: - response.raise_for_status() - user_twitter = json.loads(response.text) - self.bio_text = ( - f'{self.bio_text}{user_twitter["description"]}' - if self.twitter_bio - else f"{self.bio_text}" - ) - # Check if user has profile image - if "profile_image_url_https" in user_twitter.keys(): - self.profile_image_url = user_twitter["profile_image_url_https"] - # Check if user has banner image - if "profile_banner_url" in user_twitter.keys(): - base_banner_url = user_twitter["profile_banner_url"] - self.profile_banner_url = f"{base_banner_url}/1500x500" - self.display_name = user_twitter["name"] + for t_user in self.twitter_username: + url = f"{self.twitter_base_url_v2}/users/by/username/{t_user}" + params = {} + params.update( + { + "user.fields": "created_at,description,entities,id,location," + "name,pinned_tweet_id,profile_image_url," + "protected,url,username,verified,withheld", + "expansions": "pinned_tweet_id", + "tweet.fields": "attachments,author_id," + "context_annotations,conversation_id," + "created_at,entities," + "geo,id,in_reply_to_user_id,lang," + "public_metrics," + "possibly_sensitive,referenced_tweets," + "source,text," + "withheld", + } + ) + response = requests.get( + url, headers=self.header_twitter, auth=self.auth, params=params + ) + if not response.ok: + response.raise_for_status() + user = json.loads(response.text)["data"] + self.bio_text[t_user] = ( + f"{self.bio_text['_generic_bio_text']}{user['description']}" + if self.twitter_bio + else f"{self.bio_text['_generic_bio_text']}" + ) + # Check if user has profile image + if "profile_image_url" in user.keys(): + # Get the highest quality possible + profile_img_url = user["profile_image_url"].replace("_normal", "") + self.profile_image_url[t_user] = profile_img_url + self.display_name[t_user] = user["name"] + self.twitter_ids[user["id"]] = user["username"] + # TODO: Migrate to v2 when profile_banner is available users endpoint + twitter_user_url = ( + f"{self.twitter_base_url}" + f"/users/show.json?screen_name=" + f"{t_user}" + ) + response = requests.get( + twitter_user_url, headers=self.header_twitter, auth=self.auth + ) + if not response.ok: + response.raise_for_status() + user = json.loads(response.text) + # Check if user has banner image + if "profile_banner_url" in user.keys(): + base_banner_url = user["profile_banner_url"] + self.profile_banner_url[t_user] = f"{base_banner_url}/1500x500" return -def _get_tweets(self, version: str, tweet_id=None, start_time=None): +def _get_tweets( + self, + version: str, + tweet_id=None, + start_time=None, + t_user=None): """Gathers last 'max_tweets' tweets from the user and returns them as an dict :param version: Twitter API version to use to retrieve the tweets @@ -70,22 +104,26 @@ def _get_tweets(self, version: str, tweet_id=None, start_time=None): tweet = json.loads(response.text) return tweet else: - twitter_status_url = ( - f"{self.twitter_base_url}" - f"/statuses/user_timeline.json?screen_name=" - f"{self.twitter_username}" - f"&count={str(self.max_tweets)}&include_rts=true" - ) - response = requests.get( - twitter_status_url, headers=self.header_twitter, auth=self.auth - ) - if not response.ok: - response.raise_for_status() - tweets = json.loads(response.text) + for t_user in self.twitter_username: + twitter_status_url = ( + f"{self.twitter_base_url}" + f"/statuses/user_timeline.json?screen_name=" + f"{t_user}" + f"&count={str(self.max_tweets)}&include_rts=true" + ) + response = requests.get( + twitter_status_url, + headers=self.header_twitter, + auth=self.auth + ) + if not response.ok: + response.raise_for_status() + tweets = json.loads(response.text) + return tweets elif version == "v2": tweets_v2 = self._get_tweets_v2( - tweet_id=tweet_id, start_time=start_time + tweet_id=tweet_id, start_time=start_time, t_user=t_user ) return tweets_v2 else: @@ -98,23 +136,68 @@ def _get_tweets_v2( tweet_id=None, next_token=None, previous_token=None, + count=0, tweets_v2=None, + t_user=None ): - # Tweet number must be between 10 and 100 - if not (100 >= self.max_tweets > 10): + if not (3200 >= self.max_tweets > 10): global _ error_msg = _( - "max_tweets must be between 10 and 100. max_tweets: {}" + "max_tweets must be between 10 and 3200. max_tweets: {}" ).format(self.max_tweets) raise ValueError(error_msg) params = {} previous_token = next_token + max_tweets = self.max_tweets + diff = max_tweets - count + if diff == 0 or diff < 0: + return tweets_v2 + # Tweet number must be between 10 and 100 for search + if count: + + max_results = diff if diff < 100 else 100 + else: + max_results = max_tweets if 100 > self.max_tweets > 10 else 100 + # round up max_results to the nearest 10 + max_results = (max_results + 9) // 10 * 10 if tweet_id: url = f"{self.twitter_base_url_v2}/tweets/{tweet_id}" + params.update( + { + "poll.fields": "duration_minutes,end_datetime,id,options," + "voting_status", + "media.fields": "duration_ms,height,media_key," + "preview_image_url,type,url,width," + "public_metrics", + "expansions": "attachments.poll_ids," + "attachments.media_keys,author_id," + "entities.mentions.username,geo.place_id," + "in_reply_to_user_id,referenced_tweets.id," + "referenced_tweets.id.author_id", + "tweet.fields": "attachments,author_id," + "context_annotations,conversation_id," + "created_at,entities," + "geo,id,in_reply_to_user_id,lang," + "public_metrics," + "possibly_sensitive,referenced_tweets," + "source,text," + "withheld", + } + ) + response = requests.get( + url, headers=self.header_twitter, auth=self.auth, params=params + ) + + if not response.ok: + response.raise_for_status() + response = json.loads(response.text) + return response else: + params.update({"max_results": max_results}) + url = ( f"{self.twitter_base_url_v2}/users/by?" - f"usernames={self.twitter_username}" + f"usernames={t_user}" ) response = requests.get( url, headers=self.header_twitter, auth=self.auth @@ -122,16 +205,15 @@ def _get_tweets_v2( if not response.ok: response.raise_for_status() response = json.loads(response.text) - twitter_id = response["data"][0]["id"] + twitter_user_id = response["data"][0]["id"] - url = f"{self.twitter_base_url_v2}/users/{twitter_id}/tweets" + url = f"{self.twitter_base_url_v2}/users/{twitter_user_id}/tweets" if next_token: params.update({"pagination_token": next_token}) params.update( { "start_time": start_time, - "max_results": self.max_tweets, } ) @@ -165,6 +247,7 @@ def _get_tweets_v2( response.raise_for_status() if tweets_v2: + # TODO: Tidy up this mess next_tweets = response.json() includes = ["users", "tweets", "media", "polls"] @@ -194,12 +277,15 @@ def _get_tweets_v2( try: next_token = response.json()["meta"]["next_token"] + count += response.json()["meta"]["result_count"] if next_token and next_token != previous_token: self._get_tweets_v2( start_time=start_time, tweets_v2=tweets_v2, next_token=next_token, previous_token=previous_token, + count=count, + t_user=t_user ) except KeyError: pass @@ -209,4 +295,48 @@ def _get_tweets_v2( @spinner(_("Gathering tweets... ")) def get_tweets(self, start_time): - return self._get_tweets("v2", start_time=start_time) + t_utweets = {} + self.result_count = 0 + tweets_merged = { + "data": [], + "includes": {}, + } + try: + for t_user in self.twitter_username: + t_utweets[t_user] = self._get_tweets( + "v2", + start_time=start_time, + t_user=t_user + ) + self.result_count += t_utweets[t_user]["meta"]["result_count"] + tweets_merged["meta"] = {} + + for user in t_utweets: + includes = ["users", "tweets", "media", "polls"] + for include in includes: + try: + _ = t_utweets[user]["includes"][include] + except KeyError: + t_utweets[user]["includes"].update({include: []}) + _ = t_utweets[user]["includes"][include] + for include in includes: + try: + _ = tweets_merged["includes"][include] + except KeyError: + tweets_merged["includes"].update({include: []}) + _ = tweets_merged["includes"][include] + + tweets_merged["data"].extend(t_utweets[user]["data"]) + for in_user in t_utweets[user]["includes"]["users"]: + tweets_merged["includes"]["users"].append(in_user) # pragma + for tweet_include in t_utweets[user]["includes"]["tweets"]: + tweets_merged["includes"]["tweets"].append(tweet_include) + for media in t_utweets[user]["includes"]["media"]: + tweets_merged["includes"]["media"].append(media) + for poll in t_utweets[user]["includes"]["polls"]: + tweets_merged["includes"]["polls"].append(poll) + tweets_merged["meta"][user] = t_utweets[user]["meta"] + except KeyError: + pass + + return tweets_merged diff --git a/pleroma_bot/_utils.py b/pleroma_bot/_utils.py index 00ec413..562bf0f 100644 --- a/pleroma_bot/_utils.py +++ b/pleroma_bot/_utils.py @@ -1,18 +1,25 @@ import os import re +import sys import time import json import string import random +import zipfile +import tempfile import requests import threading import functools import mimetypes -from queue import Queue +from typing import cast +from errno import ENOENT from itertools import cycle +from multiprocessing import Queue from json.decoder import JSONDecodeError from datetime import datetime, timedelta +from itertools import tee, islice, chain +from requests.structures import CaseInsensitiveDict # Try to import libmagic # if it fails just use mimetypes @@ -21,40 +28,24 @@ except ImportError: magic = None +if sys.platform == "win32": # pragma: win32 cover + import msvcrt # pragma: win32 cover +else: + try: + import fcntl + except ImportError: # pragma: win32 cover + pass # pragma: win32 cover + from .i18n import _ from . import logger +from ._error import TimeoutLocker -class PropagatingThread(threading.Thread): - """ - Thread that surfaces exceptions that occur inside of it - """ - - def run(self): - self.exc = None - # Event to keep track if thread has started - self.started_evt = threading.Event() - try: - self.ret = self._target(*self._args, **self._kwargs) - self.started_evt.set() - except BaseException as e: - self.exc = e - self.started_evt.clear() - - def join(self): - if not self.exc: - self.started_evt.wait() - super(PropagatingThread, self).join() - self.started_evt.clear() - if self.exc: - raise self.exc - return self.ret - - -def spinner(message, spinner_symbols: list = None): +def spinner(message, wait: float = 0.3, spinner_symbols: list = None): """ Decorator that launches the function wrapped and the spinner each in a separate thread + :param wait: time to wait between symbol cycles :param message: Message to display next to the spinner :param spinner_symbols: :return: @@ -70,7 +61,7 @@ def start(): "\r{message} {symbol}".format(message=message, symbol=symbol), end="", ) - time.sleep(0.3) + time.sleep(wait) def stop(): print("\r", end="") @@ -102,13 +93,229 @@ def wrapper(*args, **kwargs): return external +def chunkify(lst, n): + return [lst[i::n] for i in range(n)] + + +def process_parallel(tweets, user, threads): + dt = tweets["data"] + chunks = chunkify(dt, threads) + mp = Multiprocessor() + for idx in range(threads): + tweets_chunked = { + "data": chunks[idx], + "includes": tweets["includes"], + "meta": tweets["meta"] + } + mp.run(user.process_tweets, tweets_chunked) + ret = mp.wait() # get all results + tweets_merged = { + "data": [], + "includes": tweets["includes"], + "meta": tweets["meta"] + } + for idx in range(threads): + tweets_merged["data"].extend(ret[idx]["data"]) + tweets_merged["data"] = sorted( + tweets_merged["data"], key=lambda i: i["created_at"] + ) + return tweets_merged + + +class Multiprocessor: + + def __init__(self): + self.processes = [] + self.queue = Queue() + + @staticmethod + def _wrapper(func, queue, args, kwargs): + ret = func(*args, **kwargs) + queue.put(ret) + + def run(self, func, *args, **kwargs): + args2 = [func, self.queue, args, kwargs] + p = PropagatingThread(target=self._wrapper, args=args2) + self.processes.append(p) + p.start() + + @spinner(_("Processing tweets... "), 1.2) + def wait(self): + rets = [] + for p in self.processes: + ret = self.queue.get() + rets.append(ret) + for p in self.processes: + p.join() + return rets + + +class PropagatingThread(threading.Thread): + """ + Thread that surfaces exceptions that occur inside of it + """ + + def run(self): + self.exc = None + # Event to keep track if thread has started + self.started_evt = threading.Event() + try: + self.ret = self._target(*self._args, **self._kwargs) + self.started_evt.set() + except BaseException as e: + self.exc = e + self.started_evt.clear() + + def join(self): + if not self.exc: + self.started_evt.wait() + super(PropagatingThread, self).join() + self.started_evt.clear() + if self.exc: + raise self.exc + return self.ret + + +class Locker: + """ + Context manager that creates lock file + """ + def __init__(self, timeout=5): + module_name = __loader__.name.split('.')[0] + lock_filename = f"{module_name}.lock" + self.timeout = timeout + self.tmp = tempfile.gettempdir() + self.mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC + self._lock_file = os.path.join(self.tmp, lock_filename) + self._lock_file_fd = None + + @property + def is_locked(self): + return self._lock_file_fd is not None + + def acquire(self): + lock_id = id(self) + lock_filename = self._lock_file + start_time = time.time() + poll_interval = 1.0 + while True: + if not self.is_locked: + logger.debug( + _( + "Attempting to acquire lock {} on {}" + ).format(lock_id, lock_filename) + ) + self._acquire() + + if self.is_locked: + logger.debug( + _("Lock {} acquired on {}").format(lock_id, lock_filename) + ) + break + elif 0 <= self.timeout < time.time() - start_time: + logger.debug( + _( + "Timeout on acquiring lock {} on {}" + ).format(lock_id, lock_filename) + ) + raise TimeoutLocker(self._lock_file) + else: + msg = _( + "Lock {} not acquired on {}, waiting {} seconds ..." + ) + logger.info( + msg.format(lock_id, lock_filename, poll_interval) + ) + time.sleep(poll_interval) + + def _acquire(self): + if sys.platform == "win32": # pragma: win32 cover + try: + fd = os.open(self._lock_file, self.mode) + except OSError as exception: + if exception.errno == ENOENT: # No such file or directory + raise + else: + try: + msvcrt.locking(fd, msvcrt.LK_NBLCK, 1) + except OSError: + os.close(fd) + else: + self._lock_file_fd = fd + else: + fd = os.open(self._lock_file, self.mode) + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + os.close(fd) + else: + self._lock_file_fd = fd + + def release(self): + if self.is_locked: # pragma: no cover + lock_id, lock_filename = id(self), self._lock_file + logger.debug( + _( + "Attempting to release lock {} on {}" + ).format(lock_id, lock_filename) + ) + self._release() + logger.debug( + _("Lock {} released on {}").format(lock_id, lock_filename) + ) + + def _release(self): + if sys.platform == "win32": # pragma: win32 cover + fd = cast(int, self._lock_file_fd) + self._lock_file_fd = None + msvcrt.locking(fd, msvcrt.LK_UNLCK, 1) + os.close(fd) + try: + os.remove(self._lock_file) + except OSError: + pass + else: + import fcntl + fd = cast(int, self._lock_file_fd) + self._lock_file_fd = None + fcntl.flock(fd, fcntl.LOCK_UN) + os.close(fd) + try: + os.remove(self._lock_file) + except OSError: # pragma: no cover + pass + + def __enter__(self): + self.acquire() + return self + + def __exit__(self, _type, value, tb): + self._release() + + def __del__(self) -> None: + """Called when the lock object is deleted.""" + self.release() + + +def previous_and_next(some_iterable): + prevs, items, nexts = tee(some_iterable, 3) + prevs = chain([None], prevs) + nexts = chain(islice(nexts, 1, None), [None]) + return zip(prevs, items, nexts) + + def check_pinned(self): """ Checks if a tweet is pinned and needs to be retrieved and posted on the Fediverse account """ - logger.info(_("Current pinned:\t{}").format(str(self.pinned_tweet_id))) - pinned_file = os.path.join(self.user_path, "pinned_id.txt") + # Only check pinned for 1 user + t_user = self.twitter_username[0] + + logger.info(_( + "Current pinned:\t{}" + ).format(str(self.pinned_tweet_id))) + pinned_file = os.path.join(self.user_path[t_user], "pinned_id.txt") if os.path.isfile(pinned_file): with open(pinned_file, "r") as file: previous_pinned_tweet_id = file.readline().rstrip() @@ -120,8 +327,8 @@ def check_pinned(self): _("Previous pinned:\t{}").format(str(previous_pinned_tweet_id)) ) if ( - self.pinned_tweet_id != previous_pinned_tweet_id - and self.pinned_tweet_id is not None + self.pinned_tweet_id != previous_pinned_tweet_id + and self.pinned_tweet_id is not None ): pinned_tweet = self._get_tweets("v2", self.pinned_tweet_id) tweets_to_post = { @@ -143,14 +350,20 @@ def check_pinned(self): file.write(f"{self.pinned_tweet_id}\n") if pleroma_pinned_post is not None: with open( - os.path.join(self.user_path, "pinned_id_pleroma.txt"), "w" + os.path.join( + self.user_path[t_user], + "pinned_id_pleroma.txt" + ), "w" ) as file: file.write(f"{pleroma_pinned_post}\n") elif ( - self.pinned_tweet_id != previous_pinned_tweet_id - and previous_pinned_tweet_id is not None + self.pinned_tweet_id != previous_pinned_tweet_id + and previous_pinned_tweet_id is not None ): - pinned_file = os.path.join(self.user_path, "pinned_id_pleroma.txt") + pinned_file = os.path.join( + self.user_path[t_user], + "pinned_id_pleroma.txt" + ) self.unpin_pleroma(pinned_file) @@ -191,6 +404,12 @@ def replace_vars_in_str(self, text: str, var_name: str = None) -> str: value = globals()[match.strip()] if isinstance(value, list): value = ", ".join([str(elem) for elem in value]) + if isinstance(value, dict) or isinstance(value, CaseInsensitiveDict): + if isinstance(self.twitter_username, list): + for t_user in self.twitter_username: + dict_value = value[t_user] + value = dict_value + # value = json.dumps(value) text = re.sub(pattern, value, text) return text @@ -238,13 +457,14 @@ def _get_instance_info(self): raise ValueError(msg) if "Pleroma" not in instance_info["version"]: logger.debug(_("Assuming target instance is Mastodon...")) - if len(self.display_name) > 30: - self.display_name = self.display_name[:30] - log_msg = _( - "Mastodon doesn't support display names longer than 30 " - "characters, truncating it and trying again..." - ) - logger.warning(log_msg) + for t_user in self.twitter_username: + if len(self.display_name[t_user]) > 30: + self.display_name[t_user] = self.display_name[t_user][:30] + log_msg = _( + "Mastodon doesn't support display names longer than 30 " + "characters, truncating it and trying again..." + ) + logger.warning(log_msg) if hasattr(self, "rich_text"): if self.rich_text: self.rich_text = False @@ -273,13 +493,68 @@ def force_date(self): datetime.now() - timedelta(days=2), "%Y-%m-%dT%H:%M:%SZ" ) elif input_date is None or input_date == "": - self.max_tweets = 100 + self.max_tweets = 3200 + logger.warning(_("Raising max_tweets to the maximum allowed value")) # Minimum date allowed - date = "2010-11-06T00:00:00Z" + tw_oldest = "2006-07-15T00:00:00Z" + tw_api_oldest = "2010-11-06T00:00:00Z" + date = tw_oldest if self.archive else tw_api_oldest else: - self.max_tweets = 100 + self.max_tweets = 3200 + logger.warning(_("Raising max_tweets to the maximum allowed value")) date = datetime.strftime( datetime.strptime(input_date, "%Y-%m-%d"), "%Y-%m-%dT%H:%M:%SZ", ) return date + + +def process_archive(archive_zip_path, start_time=None): + archive_zip_path = os.path.abspath(archive_zip_path) + par_dir = os.path.dirname(archive_zip_path) + archive_name = os.path.basename(archive_zip_path).split('.')[0] + extracted_dir = os.path.join(par_dir, archive_name) + with zipfile.ZipFile(archive_zip_path, "r") as zip_ref: + zip_ref.extractall(extracted_dir) + tweet_js_path = os.path.join(extracted_dir, 'data', 'tweet.js') + tweets_archive = get_tweets_from_archive(tweet_js_path) + tweets = { + "data": [], + "includes": { + "users": [], + "tweets": [], + "media": [], + "polls": [] + }, + "meta": {} + } + for tweet in tweets_archive: + created_at = tweet["tweet"]["created_at"] + created_at_f = datetime.strftime( + datetime.strptime(created_at, "%a %b %d %H:%M:%S +0000 %Y"), + "%Y-%m-%dT%H:%M:%SZ", + ) + if start_time: + if created_at_f < start_time: + continue + tweet["tweet"]["text"] = tweet["tweet"]["full_text"] + tweet["tweet"]["created_at"] = created_at_f + if "possibly_sensitive" not in tweet["tweet"].keys(): + tweet["tweet"]["possibly_sensitive"] = False + tweets["data"].append(tweet["tweet"]) + # Order it just in case + tweets["data"] = sorted( + tweets["data"], key=lambda i: i["created_at"], reverse=True + ) + return tweets + + +def get_tweets_from_archive(tweet_js_path): + with open(tweet_js_path, "r") as f: + lines = [] + for line in f: + line = re.sub(r'\n', r'', line) + lines.append(line) + tweets = '\n'.join(lines[1:-1]) + json_t = json.loads('{"tweets":[' + tweets + ']}') + return json_t["tweets"] diff --git a/pleroma_bot/cli.py b/pleroma_bot/cli.py index 2d25dd5..076ee16 100644 --- a/pleroma_bot/cli.py +++ b/pleroma_bot/cli.py @@ -2,7 +2,7 @@ # MIT License # -# Copyright (c) 2021 Roberto Chamorro / project contributors +# Copyright (c) 2022 Roberto Chamorro / project contributors # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -35,12 +35,15 @@ import shutil import logging import argparse +import multiprocessing as mp from requests_oauthlib import OAuth1 +from requests.structures import CaseInsensitiveDict from .i18n import _ from . import logger from .__init__ import __version__ +from ._utils import process_parallel, Locker, process_archive class User(object): @@ -68,9 +71,9 @@ class User(object): from ._processing import process_tweets from ._processing import _expand_urls + from ._processing import _replace_url from ._processing import _get_media_url from ._processing import _process_polls - from ._processing import _replace_nitter from ._processing import _download_media from ._processing import _replace_mentions from ._processing import _get_best_bitrate_video @@ -79,10 +82,12 @@ def __init__(self, user_cfg: dict, cfg: dict, base_path: str): self.posts = None self.tweets = None self.first_time = False - self.display_name = None + self.display_name = {} self.last_post_pleroma = None - self.profile_image_url = None - self.profile_banner_url = None + self.profile_image_url = {} + self.profile_banner_url = {} + self.t_user_tweets = {} + self.twitter_ids = {} valid_visibility = ("public", "unlisted", "private", "direct") default_cfg_attributes = { "twitter_base_url": "https://api.twitter.com/1.1", @@ -92,7 +97,7 @@ def __init__(self, user_cfg: dict, cfg: dict, base_path: str): "nitter": False, "twitter_token": None, "signature": False, - "media_upload": False, + "media_upload": True, "sensitive": False, "max_tweets": 50, "delay_post": 0.5, @@ -109,6 +114,12 @@ def __init__(self, user_cfg: dict, cfg: dict, base_path: str): "access_token_secret": None, "original_date": False, "original_date_format": "%Y-%m-%d %H:%M", + "bio_text": "", + "keep_media_links": False, + "fields": [], + "invidious": False, + "invidious_base_url": "https://yewtu.be/", + "archive": None, } # iterate attrs defined in config for attribute in default_cfg_attributes: @@ -119,10 +130,15 @@ def __init__(self, user_cfg: dict, cfg: dict, base_path: str): for user_attribute in user_cfg: self.__setattr__(user_attribute, user_cfg[user_attribute]) + t_users = self.twitter_username + t_users_list = isinstance(t_users, list) + t_users = [t_users] if not t_users_list else t_users + self.twitter_username = t_users + twitter_url = ( self.nitter_base_url if self.nitter else "http://twitter.com" ) - self.twitter_url = f"{twitter_url}/{self.twitter_username}" + if self.rich_text: self.content_type = "text/markdown" if not self.pleroma_base_url: @@ -135,12 +151,10 @@ def __init__(self, user_cfg: dict, cfg: dict, base_path: str): ", ".join(valid_visibility) ) ) - try: - self.fields = self.replace_vars_in_str(str(user_cfg["fields"])) - self.fields = eval(self.fields) - except KeyError: - self.fields = [] - self.bio_text = self.replace_vars_in_str(str(user_cfg["bio_text"])) + + bio_text = self.replace_vars_in_str(str(self.bio_text)) + self.bio_text = {"_generic_bio_text": bio_text} + # Auth self.header_pleroma = {"Authorization": f"Bearer {self.pleroma_token}"} self.header_twitter = {"Authorization": f"Bearer {self.twitter_token}"} @@ -168,19 +182,27 @@ def __init__(self, user_cfg: dict, cfg: dict, base_path: str): ) ) + self.twitter_url = CaseInsensitiveDict() + for t_user in t_users: + self.twitter_url[t_user] = f"{twitter_url}/{t_user}" self.pinned_tweet_id = self._get_pinned_tweet_id() - - # Filesystem + self.fields = self.replace_vars_in_str(str(self.fields)) + self.fields = eval(self.fields) self.base_path = base_path self.users_path = os.path.join(self.base_path, "users") - self.users_path = os.path.join(self.base_path, "users") - self.user_path = os.path.join(self.users_path, self.twitter_username) - self.tweets_temp_path = os.path.join(self.user_path, "tweets") - self.avatar_path = os.path.join(self.user_path, "profile.jpg") - self.header_path = os.path.join(self.user_path, "banner.jpg") + self.tweets_temp_path = os.path.join(self.base_path, "tweets") + self.user_path = {} + self.avatar_path = {} + self.header_path = {} + for t_user in t_users: + self.user_path[t_user] = os.path.join(self.users_path, t_user) + t_path = self.user_path[t_user] + self.avatar_path[t_user] = os.path.join(t_path, "profile.jpg") + self.header_path[t_user] = os.path.join(t_path, "banner.jpg") os.makedirs(self.users_path, exist_ok=True) - os.makedirs(self.user_path, exist_ok=True) os.makedirs(self.tweets_temp_path, exist_ok=True) + for t_user in t_users: + os.makedirs(self.user_path[t_user], exist_ok=True) # Get Twitter info on instance creation self._get_twitter_info() self._get_instance_info() @@ -265,6 +287,19 @@ def get_args(sysargs): help=(_("skips first run checks")), ) + parser.add_argument( + "-a", + "--archive", + required=False, + action="store", + help=( + _( + "path of the Twitter archive file (zip) to use for posting " + "tweets." + ) + ), + ) + parser.add_argument("--verbose", "-v", action="count", default=0) parser.add_argument( @@ -294,26 +329,27 @@ def main(): base_path, cfg_file = os.path.split(os.path.abspath(config_path)) else: config_path = os.path.join(base_path, "config.yml") - + tweets_temp_path = os.path.join(base_path, "tweets") + logger.info(_("config path: {}").format(config_path)) + logger.info(_("tweets temp folder: {}").format(tweets_temp_path)) + # TODO: Add config generator wizard if config file is not found? + # create a minimal config asking the user for the values with open(config_path, "r") as stream: config = yaml.safe_load(stream) user_dict = config["users"] users_path = os.path.join(base_path, "users") - # TODO: Merge tweets of multiple accounts and order them by date for user_item in user_dict[:]: user_item["skip_pin"] = False - if isinstance(user_item["twitter_username"], list): + t_users = user_item["twitter_username"] + t_user_list = isinstance(t_users, list) + t_users = t_users if t_user_list else [t_users] + if len(t_users) > 1: warn_msg = _( "Multiple twitter users for one Fediverse account, " "skipping profile and pinned tweet." ) logger.warning(warn_msg) user_item["skip_pin"] = True - for twitter_user in user_item["twitter_username"]: - new_user = dict(user_item) - new_user["twitter_username"] = twitter_user - user_dict.append(new_user) - user_dict.remove(user_item) for user_item in user_dict: first_time = False @@ -321,16 +357,22 @@ def main(): logger.info( _("Processing user:\t{}").format(user_item["pleroma_username"]) ) - user_path = os.path.join(users_path, user_item["twitter_username"]) - - if not os.path.exists(user_path): - first_time_msg = _( - "It seems like pleroma-bot is running for the " - "first time for this user" - ) - logger.info(first_time_msg) - first_time = True + t_users = user_item["twitter_username"] + t_user_list = isinstance(t_users, list) + t_users = t_users if t_user_list else [t_users] + for t_user in t_users: + user_path = os.path.join(users_path, t_user) + + if not os.path.exists(user_path): + first_time_msg = _( + "It seems like pleroma-bot is running for the " + "first time for this Twitter user: {}" + ).format(t_user) + logger.info(first_time_msg) + first_time = True user = User(user_item, config, base_path) + if args.archive: + user.archive = args.archive if first_time and not args.skipChecks: user.first_time = True if ( @@ -348,6 +390,7 @@ def main(): "includes": {}, "meta": {"result_count": len(user.tweet_ids)}, } + user.result_count = len(user.tweet_ids) includes = ["users", "tweets", "media", "polls"] for include in includes: try: @@ -372,6 +415,9 @@ def main(): tweets["includes"]["media"].append(media) for poll in next_tweet["includes"]["polls"]: tweets["includes"]["polls"].append(poll) + elif args.archive: + tweets = process_archive(args.archive, start_time=date_pleroma) + user.result_count = len(tweets["data"]) else: tweets = user.get_tweets(start_time=date_pleroma) logger.debug(f"tweets: \t {tweets}") @@ -385,14 +431,20 @@ def main(): "- access_token_secret" ) logger.error(error_msg) - - if tweets["meta"]["result_count"] > 0: + if user.result_count > 0: logger.info( - _("tweet count: \t {}").format(len(tweets["data"])) + _("tweets gathered: \t {}").format(len(tweets["data"])) ) # Put oldest first to iterate them and post them in order tweets["data"].reverse() - tweets_to_post = user.process_tweets(tweets) + cores = mp.cpu_count() + threads = round(cores / 2 if cores > 4 else 4) + tweets_to_post = process_parallel(tweets, user, threads) + logger.info( + _("tweets to post: \t {}").format( + len(tweets_to_post['data']) + ) + ) logger.debug(f"tweets_processed: \t {tweets_to_post['data']}") tweet_counter = 0 for tweet in tweets_to_post["data"]: @@ -448,7 +500,8 @@ def init(): f_handler.setFormatter(f_format) logger.addHandler(f_handler) if __name__ == "__main__": - sys.exit(main()) + with Locker(): + sys.exit(main()) init() diff --git a/pleroma_bot/locale/es/LC_MESSAGES/pleroma_bot.mo b/pleroma_bot/locale/es/LC_MESSAGES/pleroma_bot.mo index c293b7a..a65ffa5 100644 Binary files a/pleroma_bot/locale/es/LC_MESSAGES/pleroma_bot.mo and b/pleroma_bot/locale/es/LC_MESSAGES/pleroma_bot.mo differ diff --git a/pleroma_bot/locale/es/LC_MESSAGES/pleroma_bot.po b/pleroma_bot/locale/es/LC_MESSAGES/pleroma_bot.po index 80c47b7..12d8e75 100644 --- a/pleroma_bot/locale/es/LC_MESSAGES/pleroma_bot.po +++ b/pleroma_bot/locale/es/LC_MESSAGES/pleroma_bot.po @@ -5,8 +5,8 @@ msgid "" msgstr "" "Project-Id-Version: pleroma-bot\n" -"POT-Creation-Date: 2021-12-04 22:42+0100\n" -"PO-Revision-Date: 2021-12-04 22:45+0100\n" +"POT-Creation-Date: 2022-01-09 00:15+0100\n" +"PO-Revision-Date: 2022-01-09 00:15+0100\n" "Last-Translator: robertoszek \n" "Language-Team: \n" "Language: es\n" @@ -20,15 +20,23 @@ msgstr "" "X-Poedit-SearchPath-0: .\n" "X-Poedit-SearchPathExcluded-0: tests\n" -#: _pin.py:25 +#: _error.py:13 +msgid "" +"The file lock '{}' could not be acquired. Is another instance of pleroma-bot " +"running?" +msgstr "" +"El bloqueo del fichero '{}' no pudo ser adquirido. ยฟHay otra instancia de " +"pleroma-bot en ejecuciรณn?" + +#: _pin.py:28 msgid "Pinning post:\t{}" msgstr "Fijando post:\t{}" -#: _pin.py:56 +#: _pin.py:62 msgid "Unpinning previous:\t{}" msgstr "Desfijando anterior:\t{}" -#: _pin.py:60 +#: _pin.py:66 msgid "" "File with previous pinned post ID not found or empty. Checking last posts " "for pinned post..." @@ -36,7 +44,7 @@ msgstr "" "No se ha encontrado el archivo con el ID del anterior post fijado. Revisando " "รบltimos posts..." -#: _pin.py:65 +#: _pin.py:71 msgid "Pinned post not found. Giving up unpinning..." msgstr "Post fijado no encontrado. Dejamos de intentar el desfijado..." @@ -44,7 +52,7 @@ msgstr "Post fijado no encontrado. Dejamos de intentar el desfijado..." msgid "No posts were found in the target Fediverse account" msgstr "No se han encontrado posts en la cuenta de Fediverso objectivo" -#: _pleroma.py:105 +#: _pleroma.py:107 #, python-brace-format msgid "" "Exception occurred\n" @@ -61,15 +69,15 @@ msgstr "" "Considere aumentar el lรญmite de\n" " tamaรฑo de adjuntos para su instancia" -#: _pleroma.py:120 +#: _pleroma.py:122 msgid "Error uploading media:\t{}" msgstr "Error mientras subiendo contenido:\t{}" -#: _pleroma.py:160 +#: _pleroma.py:155 msgid "Post in Pleroma:\t{}" msgstr "Publicando en Pleroma\t{}" -#: _pleroma.py:216 +#: _pleroma.py:219 msgid "" "Total number of metadata fields cannot exceed 4.\n" "Provided: {}. Exiting..." @@ -77,7 +85,7 @@ msgstr "" "El nรบmero de campos de metadatos no puede ser superior a 4.\n" "Se han introducido {}. Saliendo..." -#: _pleroma.py:254 +#: _pleroma.py:257 msgid "" "Exception occurred\n" "Error code 422\n" @@ -92,15 +100,11 @@ msgstr "" "metadatos\n" "no se muy largo." -#: _pleroma.py:264 +#: _pleroma.py:267 msgid "Updating profile:\t {}" msgstr "Actualizando perfil:\t{}" -#: _processing.py:22 -msgid "Processing tweets... " -msgstr "Procesando tweets... " - -#: _processing.py:155 +#: _processing.py:199 #, python-brace-format msgid "" "Exception occurred\n" @@ -113,47 +117,79 @@ msgstr "" "{tweet} - {media_url}\n" "Ignorando el adjunto y continuando..." -#: _processing.py:180 +#: _processing.py:224 msgid "Attachment exceeded config file size limit ({})" msgstr "Contenido adjunto ha excedido el lรญmite configurado ({})" -#: _processing.py:184 +#: _processing.py:228 msgid "File size: {}MB" msgstr "Tamaรฑo de fichero: {}MB" -#: _processing.py:188 +#: _processing.py:232 msgid "Ignoring attachment and continuing..." msgstr "Ignorando adjunto y continuando..." -#: _twitter.py:92 +#: _processing.py:315 +msgid "Couldn't expand the {}: {}" +msgstr "No se pudo expandir la URL {}: {}" + +#: _twitter.py:130 msgid "API version not supported: {}" msgstr "Versiรณn de API no soportada: {}" -#: _twitter.py:107 -msgid "max_tweets must be between 10 and 100. max_tweets: {}" -msgstr "El valor de max_tweets debe estar entre 10 y 100. max_tweets: {}" +#: _twitter.py:146 +msgid "max_tweets must be between 10 and 3200. max_tweets: {}" +msgstr "El valor de max_tweets debe estar entre 10 y 3200. max_tweets: {}" -#: _twitter.py:210 +#: _twitter.py:296 msgid "Gathering tweets... " msgstr "Recopilando tweets... " -#: _utils.py:110 +#: _utils.py:142 +msgid "Processing tweets... " +msgstr "Procesando tweets... " + +#: _utils.py:205 +msgid "Attempting to acquire lock {} on {}" +msgstr "Intentando adquirir el bloqueo {} en {}" + +#: _utils.py:212 +msgid "Lock {} acquired on {}" +msgstr "Bloqueo {} adquirido en {}" + +#: _utils.py:218 +msgid "Timeout on acquiring lock {} on {}" +msgstr "Tiempo de espera excedido al intentar adquirir el bloqueo {} en {}" + +#: _utils.py:224 +msgid "Lock {} not acquired on {}, waiting {} seconds ..." +msgstr "Bloqueo {} no adquirido en {}, esperando {} segundos ..." + +#: _utils.py:259 +msgid "Attempting to release lock {} on {}" +msgstr "Intentando liberar el bloqueo {} en {}" + +#: _utils.py:264 +msgid "Lock {} released on {}" +msgstr "Bloqueo {} liberado en {}" + +#: _utils.py:316 msgid "Current pinned:\t{}" msgstr "Fijado actualmente:\t{}" -#: _utils.py:120 +#: _utils.py:327 msgid "Previous pinned:\t{}" msgstr "Fijado anteriormente:\t{}" -#: _utils.py:231 +#: _utils.py:454 msgid "Instance response was not understood {}" msgstr "La respuesta de la instancia no ha podido ser interpretada {}" -#: _utils.py:236 +#: _utils.py:459 msgid "Assuming target instance is Mastodon..." msgstr "Suponiendo la instancia objetivo es Mastodon..." -#: _utils.py:240 +#: _utils.py:464 msgid "" "Mastodon doesn't support display names longer than 30 characters, truncating " "it and trying again..." @@ -161,15 +197,15 @@ msgstr "" "Mastodon no soporta nombres para mostrar de mรกs de 30 caracteres, " "truncรกndolo e intentรกndolo de nuevo..." -#: _utils.py:248 +#: _utils.py:472 msgid "Mastodon doesn't support rich text. Disabling it..." msgstr "Mastodon no soporta texto enriquecido. Desactivรกndolo..." -#: _utils.py:254 +#: _utils.py:478 msgid "How far back should we retrieve tweets from the Twitter account?" msgstr "ยฟDesde cuรกndo deberรญamos recopilar los tweets de la cuenta de Twitter?" -#: _utils.py:257 +#: _utils.py:481 msgid "" "\n" "Enter a date (YYYY-MM-DD):\n" @@ -184,16 +220,20 @@ msgstr "" "comporte como de costumbre (comprobando la fecha del\n" "รบltimo post de la cuenta del Fediverso)] " -#: cli.py:128 +#: _utils.py:497 _utils.py:504 +msgid "Raising max_tweets to the maximum allowed value" +msgstr "Aumentando max_tweets al valor mรกximo admitido" + +#: cli.py:146 msgid "No Pleroma URL defined in config! [pleroma_base_url]" msgstr "" "ยกNo se ha definido la URL de Pleroma en la configuraciรณn! [pleroma_base_url]" -#: cli.py:132 +#: cli.py:150 msgid "Visibility not supported! Values allowed are: {}" msgstr "Visibilidad introducida no soportada. Valores vรกlidos: {}" -#: cli.py:164 +#: cli.py:180 msgid "" "Some or all OAuth 1.0a tokens missing, falling back to application-only " "authentication" @@ -201,11 +241,11 @@ msgstr "" "No todos los tokens OAuth 1.0a han sido encontrados. Usando autenticaciรณn de " "aplicaciรณn en su lugar" -#: cli.py:196 +#: cli.py:220 msgid "Bot for mirroring one or multiple Twitter accounts in Pleroma/Mastodon." msgstr "Bot para replicar una o varias cuentas de Twitter en Pleroma/Mastodon." -#: cli.py:209 +#: cli.py:233 msgid "" "path of config file (config.yml) to use and parse. If not specified, it will " "try to find it in the current working directory." @@ -213,7 +253,7 @@ msgstr "" "ruta del fichero de configuraciรณn (config.yml) a usar e interpretar. Si no " "se especifica, se intentarรก usar el del directorio de trabajo actual." -#: cli.py:223 +#: cli.py:247 msgid "" "path of log file (error.log) to create. If not specified, it will try to " "store it at your config file path" @@ -221,7 +261,7 @@ msgstr "" "ruta del fichero de log (error.log) a escribir. Si no se especifica, se " "intentarรก usar la ruta en la que se encuentra el fichero de configuraciรณn" -#: cli.py:236 +#: cli.py:260 msgid "" "skips Fediverse profile update (no background image, profile image, bio " "text, etc.)" @@ -229,7 +269,7 @@ msgstr "" "omite la actualizaciรณn de perfil en la cuenta del Fediverso (imagen de " "fondo, imagen de perfil, biografรญa, etc.)" -#: cli.py:250 +#: cli.py:274 msgid "" "forces the tweet retrieval to start from a specific date. The " "twitter_username value (FORCEDATE) can be supplied to only force it for that " @@ -240,15 +280,28 @@ msgstr "" "fecha de inicio de recopilaciรณn sรณlo para ese usuario especรญfico del archivo " "de configuraciรณn" -#: cli.py:263 +#: cli.py:287 msgid "skips first run checks" msgstr "omite las validaciones de la primera ejecuciรณn" -#: cli.py:286 +#: cli.py:297 +msgid "path of the Twitter archive file (zip) to use for posting tweets." +msgstr "" +"ruta del fichero de archivo de Twitter (zip) a usar para publicar tweets." + +#: cli.py:323 msgid "Debug logging enabled" msgstr "Modo de depuraciรณn activado" -#: cli.py:305 +#: cli.py:333 +msgid "config path: {}" +msgstr "ruta de configuraciรณn: {}" + +#: cli.py:334 +msgid "tweets temp folder: {}" +msgstr "carpeta temporal de tweets: {}" + +#: cli.py:348 msgid "" "Multiple twitter users for one Fediverse account, skipping profile and " "pinned tweet." @@ -256,16 +309,19 @@ msgstr "" "Varios usuarios de Twitter definidos para una cuenta de Fediverso, omitiendo " "la actualizaciรณn de perfil y tweet fijado." -#: cli.py:320 +#: cli.py:358 msgid "Processing user:\t{}" msgstr "Procesando usuario:\t{}" -#: cli.py:326 -msgid "It seems like pleroma-bot is running for the first time for this user" +#: cli.py:368 +msgid "" +"It seems like pleroma-bot is running for the first time for this Twitter " +"user: {}" msgstr "" -"Parece que pleroma-bot estรก ejecutรกndose por primera vez para este usuario" +"Parece que pleroma-bot estรก ejecutรกndose por primera vez para este usuario: " +"{}" -#: cli.py:379 +#: cli.py:427 msgid "" "Unable to retrieve tweets. Is the account protected? If so, you need to " "provide the following OAuth 1.0a fields in the user config:\n" @@ -282,14 +338,18 @@ msgstr "" " - access_token_key \n" " - access_token_secret" -#: cli.py:389 -msgid "tweet count: \t {}" -msgstr "nรบmero de tweets: \t {}" +#: cli.py:436 +msgid "tweets gathered: \t {}" +msgstr "tweets recopilados: \t {}" + +#: cli.py:444 +msgid "tweets to post: \t {}" +msgstr "tweets a publicar: \t {}" -#: cli.py:414 +#: cli.py:468 msgid "Multiple twitter users, not updating profile" msgstr "Varios usuarios de Twitter, omitiendo actualizado de perfil" -#: cli.py:421 +#: cli.py:475 msgid "Exception occurred" msgstr "Se produjo una excepciรณn" diff --git a/pleroma_bot/locale/es_ES/LC_MESSAGES/pleroma_bot.mo b/pleroma_bot/locale/es_ES/LC_MESSAGES/pleroma_bot.mo index c8600e7..45475a2 100644 Binary files a/pleroma_bot/locale/es_ES/LC_MESSAGES/pleroma_bot.mo and b/pleroma_bot/locale/es_ES/LC_MESSAGES/pleroma_bot.mo differ diff --git a/pleroma_bot/locale/es_ES/LC_MESSAGES/pleroma_bot.po b/pleroma_bot/locale/es_ES/LC_MESSAGES/pleroma_bot.po index 447b9ab..4f4646e 100644 --- a/pleroma_bot/locale/es_ES/LC_MESSAGES/pleroma_bot.po +++ b/pleroma_bot/locale/es_ES/LC_MESSAGES/pleroma_bot.po @@ -5,30 +5,38 @@ msgid "" msgstr "" "Project-Id-Version: pleroma-bot\n" -"POT-Creation-Date: 2021-12-04 22:43+0100\n" -"PO-Revision-Date: 2021-12-04 22:45+0100\n" +"POT-Creation-Date: 2022-01-09 00:13+0100\n" +"PO-Revision-Date: 2022-01-09 00:14+0100\n" "Last-Translator: robertoszek \n" "Language-Team: \n" -"Language: es_ES\n" +"Language: es\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" "Content-Transfer-Encoding: 8bit\n" "Generated-By: pygettext.py 1.5\n" "X-Generator: Poedit 3.0\n" -"Plural-Forms: nplurals=2; plural=(n != 1);\n" "X-Poedit-Basepath: ../../..\n" +"Plural-Forms: nplurals=2; plural=(n != 1);\n" "X-Poedit-SearchPath-0: .\n" "X-Poedit-SearchPathExcluded-0: tests\n" -#: _pin.py:25 +#: _error.py:13 +msgid "" +"The file lock '{}' could not be acquired. Is another instance of pleroma-bot " +"running?" +msgstr "" +"El bloqueo del fichero '{}' no pudo ser adquirido. ยฟHay otra instancia de " +"pleroma-bot en ejecuciรณn?" + +#: _pin.py:28 msgid "Pinning post:\t{}" msgstr "Fijando post:\t{}" -#: _pin.py:56 +#: _pin.py:62 msgid "Unpinning previous:\t{}" msgstr "Desfijando anterior:\t{}" -#: _pin.py:60 +#: _pin.py:66 msgid "" "File with previous pinned post ID not found or empty. Checking last posts " "for pinned post..." @@ -36,7 +44,7 @@ msgstr "" "No se ha encontrado el archivo con el ID del anterior post fijado. Revisando " "รบltimos posts..." -#: _pin.py:65 +#: _pin.py:71 msgid "Pinned post not found. Giving up unpinning..." msgstr "Post fijado no encontrado. Dejamos de intentar el desfijado..." @@ -44,7 +52,7 @@ msgstr "Post fijado no encontrado. Dejamos de intentar el desfijado..." msgid "No posts were found in the target Fediverse account" msgstr "No se han encontrado posts en la cuenta de Fediverso objectivo" -#: _pleroma.py:105 +#: _pleroma.py:107 #, python-brace-format msgid "" "Exception occurred\n" @@ -61,15 +69,15 @@ msgstr "" "Considere aumentar el lรญmite de\n" " tamaรฑo de adjuntos para su instancia" -#: _pleroma.py:120 +#: _pleroma.py:122 msgid "Error uploading media:\t{}" msgstr "Error mientras subiendo contenido:\t{}" -#: _pleroma.py:160 +#: _pleroma.py:155 msgid "Post in Pleroma:\t{}" msgstr "Publicando en Pleroma\t{}" -#: _pleroma.py:216 +#: _pleroma.py:219 msgid "" "Total number of metadata fields cannot exceed 4.\n" "Provided: {}. Exiting..." @@ -77,7 +85,7 @@ msgstr "" "El nรบmero de campos de metadatos no puede ser superior a 4.\n" "Se han introducido {}. Saliendo..." -#: _pleroma.py:254 +#: _pleroma.py:257 msgid "" "Exception occurred\n" "Error code 422\n" @@ -92,15 +100,11 @@ msgstr "" "metadatos\n" "no se muy largo." -#: _pleroma.py:264 +#: _pleroma.py:267 msgid "Updating profile:\t {}" msgstr "Actualizando perfil:\t{}" -#: _processing.py:22 -msgid "Processing tweets... " -msgstr "Procesando tweets... " - -#: _processing.py:155 +#: _processing.py:199 #, python-brace-format msgid "" "Exception occurred\n" @@ -113,47 +117,79 @@ msgstr "" "{tweet} - {media_url}\n" "Ignorando el adjunto y continuando..." -#: _processing.py:180 +#: _processing.py:224 msgid "Attachment exceeded config file size limit ({})" msgstr "Contenido adjunto ha excedido el lรญmite configurado ({})" -#: _processing.py:184 +#: _processing.py:228 msgid "File size: {}MB" msgstr "Tamaรฑo de fichero: {}MB" -#: _processing.py:188 +#: _processing.py:232 msgid "Ignoring attachment and continuing..." msgstr "Ignorando adjunto y continuando..." -#: _twitter.py:92 +#: _processing.py:315 +msgid "Couldn't expand the {}: {}" +msgstr "No se pudo expandir la URL {}: {}" + +#: _twitter.py:130 msgid "API version not supported: {}" msgstr "Versiรณn de API no soportada: {}" -#: _twitter.py:107 -msgid "max_tweets must be between 10 and 100. max_tweets: {}" -msgstr "El valor de max_tweets debe estar entre 10 y 100. max_tweets: {}" +#: _twitter.py:146 +msgid "max_tweets must be between 10 and 3200. max_tweets: {}" +msgstr "El valor de max_tweets debe estar entre 10 y 3200. max_tweets: {}" -#: _twitter.py:210 +#: _twitter.py:296 msgid "Gathering tweets... " msgstr "Recopilando tweets... " -#: _utils.py:110 +#: _utils.py:142 +msgid "Processing tweets... " +msgstr "Procesando tweets... " + +#: _utils.py:205 +msgid "Attempting to acquire lock {} on {}" +msgstr "Intentando adquirir el bloqueo {} en {}" + +#: _utils.py:212 +msgid "Lock {} acquired on {}" +msgstr "Bloqueo {} adquirido en {}" + +#: _utils.py:218 +msgid "Timeout on acquiring lock {} on {}" +msgstr "Tiempo de espera excedido al intentar adquirir el bloqueo {} en {}" + +#: _utils.py:224 +msgid "Lock {} not acquired on {}, waiting {} seconds ..." +msgstr "Bloqueo {} no adquirido en {}, esperando {} segundos ..." + +#: _utils.py:259 +msgid "Attempting to release lock {} on {}" +msgstr "Intentando liberar el bloqueo {} en {}" + +#: _utils.py:264 +msgid "Lock {} released on {}" +msgstr "Bloqueo {} liberado en {}" + +#: _utils.py:316 msgid "Current pinned:\t{}" msgstr "Fijado actualmente:\t{}" -#: _utils.py:120 +#: _utils.py:327 msgid "Previous pinned:\t{}" msgstr "Fijado anteriormente:\t{}" -#: _utils.py:231 +#: _utils.py:454 msgid "Instance response was not understood {}" msgstr "La respuesta de la instancia no ha podido ser interpretada {}" -#: _utils.py:236 +#: _utils.py:459 msgid "Assuming target instance is Mastodon..." msgstr "Suponiendo la instancia objetivo es Mastodon..." -#: _utils.py:240 +#: _utils.py:464 msgid "" "Mastodon doesn't support display names longer than 30 characters, truncating " "it and trying again..." @@ -161,15 +197,15 @@ msgstr "" "Mastodon no soporta nombres para mostrar de mรกs de 30 caracteres, " "truncรกndolo e intentรกndolo de nuevo..." -#: _utils.py:248 +#: _utils.py:472 msgid "Mastodon doesn't support rich text. Disabling it..." msgstr "Mastodon no soporta texto enriquecido. Desactivรกndolo..." -#: _utils.py:254 +#: _utils.py:478 msgid "How far back should we retrieve tweets from the Twitter account?" msgstr "ยฟDesde cuรกndo deberรญamos recopilar los tweets de la cuenta de Twitter?" -#: _utils.py:257 +#: _utils.py:481 msgid "" "\n" "Enter a date (YYYY-MM-DD):\n" @@ -184,16 +220,20 @@ msgstr "" "comporte como de costumbre (comprobando la fecha del\n" "รบltimo post de la cuenta del Fediverso)] " -#: cli.py:128 +#: _utils.py:497 _utils.py:504 +msgid "Raising max_tweets to the maximum allowed value" +msgstr "Aumentando max_tweets al valor mรกximo admitido" + +#: cli.py:146 msgid "No Pleroma URL defined in config! [pleroma_base_url]" msgstr "" "ยกNo se ha definido la URL de Pleroma en la configuraciรณn! [pleroma_base_url]" -#: cli.py:132 +#: cli.py:150 msgid "Visibility not supported! Values allowed are: {}" msgstr "Visibilidad introducida no soportada. Valores vรกlidos: {}" -#: cli.py:164 +#: cli.py:180 msgid "" "Some or all OAuth 1.0a tokens missing, falling back to application-only " "authentication" @@ -201,11 +241,11 @@ msgstr "" "No todos los tokens OAuth 1.0a han sido encontrados. Usando autenticaciรณn de " "aplicaciรณn en su lugar" -#: cli.py:196 +#: cli.py:220 msgid "Bot for mirroring one or multiple Twitter accounts in Pleroma/Mastodon." msgstr "Bot para replicar una o varias cuentas de Twitter en Pleroma/Mastodon." -#: cli.py:209 +#: cli.py:233 msgid "" "path of config file (config.yml) to use and parse. If not specified, it will " "try to find it in the current working directory." @@ -213,7 +253,7 @@ msgstr "" "ruta del fichero de configuraciรณn (config.yml) a usar e interpretar. Si no " "se especifica, se intentarรก usar el del directorio de trabajo actual." -#: cli.py:223 +#: cli.py:247 msgid "" "path of log file (error.log) to create. If not specified, it will try to " "store it at your config file path" @@ -221,7 +261,7 @@ msgstr "" "ruta del fichero de log (error.log) a escribir. Si no se especifica, se " "intentarรก usar la ruta en la que se encuentra el fichero de configuraciรณn" -#: cli.py:236 +#: cli.py:260 msgid "" "skips Fediverse profile update (no background image, profile image, bio " "text, etc.)" @@ -229,7 +269,7 @@ msgstr "" "omite la actualizaciรณn de perfil en la cuenta del Fediverso (imagen de " "fondo, imagen de perfil, biografรญa, etc.)" -#: cli.py:250 +#: cli.py:274 msgid "" "forces the tweet retrieval to start from a specific date. The " "twitter_username value (FORCEDATE) can be supplied to only force it for that " @@ -240,15 +280,28 @@ msgstr "" "fecha de inicio de recopilaciรณn sรณlo para ese usuario especรญfico del archivo " "de configuraciรณn" -#: cli.py:263 +#: cli.py:287 msgid "skips first run checks" msgstr "omite las validaciones de la primera ejecuciรณn" -#: cli.py:286 +#: cli.py:297 +msgid "path of the Twitter archive file (zip) to use for posting tweets." +msgstr "" +"ruta del fichero de archivo de Twitter (zip) a usar para publicar tweets." + +#: cli.py:323 msgid "Debug logging enabled" msgstr "Modo de depuraciรณn activado" -#: cli.py:305 +#: cli.py:333 +msgid "config path: {}" +msgstr "ruta de configuraciรณn: {}" + +#: cli.py:334 +msgid "tweets temp folder: {}" +msgstr "carpeta temporal de tweets: {}" + +#: cli.py:348 msgid "" "Multiple twitter users for one Fediverse account, skipping profile and " "pinned tweet." @@ -256,16 +309,19 @@ msgstr "" "Varios usuarios de Twitter definidos para una cuenta de Fediverso, omitiendo " "la actualizaciรณn de perfil y tweet fijado." -#: cli.py:320 +#: cli.py:358 msgid "Processing user:\t{}" msgstr "Procesando usuario:\t{}" -#: cli.py:326 -msgid "It seems like pleroma-bot is running for the first time for this user" +#: cli.py:368 +msgid "" +"It seems like pleroma-bot is running for the first time for this Twitter " +"user: {}" msgstr "" -"Parece que pleroma-bot estรก ejecutรกndose por primera vez para este usuario" +"Parece que pleroma-bot estรก ejecutรกndose por primera vez para este usuario: " +"{}" -#: cli.py:379 +#: cli.py:427 msgid "" "Unable to retrieve tweets. Is the account protected? If so, you need to " "provide the following OAuth 1.0a fields in the user config:\n" @@ -282,14 +338,18 @@ msgstr "" " - access_token_key \n" " - access_token_secret" -#: cli.py:389 -msgid "tweet count: \t {}" -msgstr "nรบmero de tweets: \t {}" +#: cli.py:436 +msgid "tweets gathered: \t {}" +msgstr "tweets recopilados: \t {}" + +#: cli.py:444 +msgid "tweets to post: \t {}" +msgstr "tweets a publicar: \t {}" -#: cli.py:414 +#: cli.py:468 msgid "Multiple twitter users, not updating profile" msgstr "Varios usuarios de Twitter, omitiendo actualizado de perfil" -#: cli.py:421 +#: cli.py:475 msgid "Exception occurred" msgstr "Se produjo una excepciรณn" diff --git a/pleroma_bot/locale/pleroma_bot.pot b/pleroma_bot/locale/pleroma_bot.pot index 6a3ae61..cddd9ea 100644 --- a/pleroma_bot/locale/pleroma_bot.pot +++ b/pleroma_bot/locale/pleroma_bot.pot @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" -"POT-Creation-Date: 2021-12-04 22:45+0100\n" +"POT-Creation-Date: 2022-01-09 00:13+0100\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -19,19 +19,23 @@ msgstr "" "X-Poedit-SearchPath-0: .\n" "X-Poedit-SearchPathExcluded-0: tests\n" -#: _pin.py:25 +#: _error.py:13 +msgid "The file lock '{}' could not be acquired. Is another instance of pleroma-bot running?" +msgstr "" + +#: _pin.py:28 msgid "Pinning post:\t{}" msgstr "" -#: _pin.py:56 +#: _pin.py:62 msgid "Unpinning previous:\t{}" msgstr "" -#: _pin.py:60 +#: _pin.py:66 msgid "File with previous pinned post ID not found or empty. Checking last posts for pinned post..." msgstr "" -#: _pin.py:65 +#: _pin.py:71 msgid "Pinned post not found. Giving up unpinning..." msgstr "" @@ -39,7 +43,7 @@ msgstr "" msgid "No posts were found in the target Fediverse account" msgstr "" -#: _pleroma.py:105 +#: _pleroma.py:107 #, python-brace-format msgid "" "Exception occurred\n" @@ -50,21 +54,21 @@ msgid "" " size limit of your instance" msgstr "" -#: _pleroma.py:120 +#: _pleroma.py:122 msgid "Error uploading media:\t{}" msgstr "" -#: _pleroma.py:160 +#: _pleroma.py:155 msgid "Post in Pleroma:\t{}" msgstr "" -#: _pleroma.py:216 +#: _pleroma.py:219 msgid "" "Total number of metadata fields cannot exceed 4.\n" "Provided: {}. Exiting..." msgstr "" -#: _pleroma.py:254 +#: _pleroma.py:257 msgid "" "Exception occurred\n" "Error code 422\n" @@ -73,15 +77,11 @@ msgid "" "aren't too long." msgstr "" -#: _pleroma.py:264 +#: _pleroma.py:267 msgid "Updating profile:\t {}" msgstr "" -#: _processing.py:22 -msgid "Processing tweets... " -msgstr "" - -#: _processing.py:155 +#: _processing.py:199 #, python-brace-format msgid "" "Exception occurred\n" @@ -90,59 +90,91 @@ msgid "" "Ignoring attachment and continuing..." msgstr "" -#: _processing.py:180 +#: _processing.py:224 msgid "Attachment exceeded config file size limit ({})" msgstr "" -#: _processing.py:184 +#: _processing.py:228 msgid "File size: {}MB" msgstr "" -#: _processing.py:188 +#: _processing.py:232 msgid "Ignoring attachment and continuing..." msgstr "" -#: _twitter.py:92 +#: _processing.py:315 +msgid "Couldn't expand the {}: {}" +msgstr "" + +#: _twitter.py:130 msgid "API version not supported: {}" msgstr "" -#: _twitter.py:107 -msgid "max_tweets must be between 10 and 100. max_tweets: {}" +#: _twitter.py:146 +msgid "max_tweets must be between 10 and 3200. max_tweets: {}" msgstr "" -#: _twitter.py:210 +#: _twitter.py:296 msgid "Gathering tweets... " msgstr "" -#: _utils.py:110 +#: _utils.py:142 +msgid "Processing tweets... " +msgstr "" + +#: _utils.py:205 +msgid "Attempting to acquire lock {} on {}" +msgstr "" + +#: _utils.py:212 +msgid "Lock {} acquired on {}" +msgstr "" + +#: _utils.py:218 +msgid "Timeout on acquiring lock {} on {}" +msgstr "" + +#: _utils.py:224 +msgid "Lock {} not acquired on {}, waiting {} seconds ..." +msgstr "" + +#: _utils.py:259 +msgid "Attempting to release lock {} on {}" +msgstr "" + +#: _utils.py:264 +msgid "Lock {} released on {}" +msgstr "" + +#: _utils.py:316 msgid "Current pinned:\t{}" msgstr "" -#: _utils.py:120 +#: _utils.py:327 msgid "Previous pinned:\t{}" msgstr "" -#: _utils.py:231 +#: _utils.py:454 msgid "Instance response was not understood {}" msgstr "" -#: _utils.py:236 +#: _utils.py:459 msgid "Assuming target instance is Mastodon..." msgstr "" -#: _utils.py:240 +#: _utils.py:464 msgid "Mastodon doesn't support display names longer than 30 characters, truncating it and trying again..." msgstr "" -#: _utils.py:248 +#: _utils.py:472 msgid "Mastodon doesn't support rich text. Disabling it..." msgstr "" -#: _utils.py:254 +#: _utils.py:478 msgid "How far back should we retrieve tweets from the Twitter account?" msgstr "" -#: _utils.py:257 +#: _utils.py:481 msgid "" "\n" "Enter a date (YYYY-MM-DD):\n" @@ -151,59 +183,75 @@ msgid "" "last post in the Fediverse account)] " msgstr "" -#: cli.py:128 +#: _utils.py:497 _utils.py:504 +msgid "Raising max_tweets to the maximum allowed value" +msgstr "" + +#: cli.py:146 msgid "No Pleroma URL defined in config! [pleroma_base_url]" msgstr "" -#: cli.py:132 +#: cli.py:150 msgid "Visibility not supported! Values allowed are: {}" msgstr "" -#: cli.py:164 +#: cli.py:180 msgid "Some or all OAuth 1.0a tokens missing, falling back to application-only authentication" msgstr "" -#: cli.py:196 +#: cli.py:220 msgid "Bot for mirroring one or multiple Twitter accounts in Pleroma/Mastodon." msgstr "" -#: cli.py:209 +#: cli.py:233 msgid "path of config file (config.yml) to use and parse. If not specified, it will try to find it in the current working directory." msgstr "" -#: cli.py:223 +#: cli.py:247 msgid "path of log file (error.log) to create. If not specified, it will try to store it at your config file path" msgstr "" -#: cli.py:236 +#: cli.py:260 msgid "skips Fediverse profile update (no background image, profile image, bio text, etc.)" msgstr "" -#: cli.py:250 +#: cli.py:274 msgid "forces the tweet retrieval to start from a specific date. The twitter_username value (FORCEDATE) can be supplied to only force it for that particular user in the config" msgstr "" -#: cli.py:263 +#: cli.py:287 msgid "skips first run checks" msgstr "" -#: cli.py:286 +#: cli.py:297 +msgid "path of the Twitter archive file (zip) to use for posting tweets." +msgstr "" + +#: cli.py:323 msgid "Debug logging enabled" msgstr "" -#: cli.py:305 +#: cli.py:333 +msgid "config path: {}" +msgstr "" + +#: cli.py:334 +msgid "tweets temp folder: {}" +msgstr "" + +#: cli.py:348 msgid "Multiple twitter users for one Fediverse account, skipping profile and pinned tweet." msgstr "" -#: cli.py:320 +#: cli.py:358 msgid "Processing user:\t{}" msgstr "" -#: cli.py:326 -msgid "It seems like pleroma-bot is running for the first time for this user" +#: cli.py:368 +msgid "It seems like pleroma-bot is running for the first time for this Twitter user: {}" msgstr "" -#: cli.py:379 +#: cli.py:427 msgid "" "Unable to retrieve tweets. Is the account protected? If so, you need to provide the following OAuth 1.0a fields in the user config:\n" " - consumer_key \n" @@ -212,14 +260,18 @@ msgid "" " - access_token_secret" msgstr "" -#: cli.py:389 -msgid "tweet count: \t {}" +#: cli.py:436 +msgid "tweets gathered: \t {}" +msgstr "" + +#: cli.py:444 +msgid "tweets to post: \t {}" msgstr "" -#: cli.py:414 +#: cli.py:468 msgid "Multiple twitter users, not updating profile" msgstr "" -#: cli.py:421 +#: cli.py:475 msgid "Exception occurred" msgstr "" diff --git a/pleroma_bot/tests/conftest.py b/pleroma_bot/tests/conftest.py index 71477ab..cbc54a9 100644 --- a/pleroma_bot/tests/conftest.py +++ b/pleroma_bot/tests/conftest.py @@ -57,7 +57,50 @@ def mock_request(rootdir): status_code=301, headers={'Location': 'http://github.com'} ) + + mock.head( + "http://cutt.ly/xg3TuYA", + status_code=301, + headers={ + 'Location': 'https://twitter.com/BotPleroma/status' + '/1474760145850806283/video/1' + } + ) + mock.head( + 'https://twitter.com/BotPleroma/status' + '/1474760145850806283/video/1', + status_code=200, + headers={ + 'Location': 'https://twitter.com/BotPleroma/status' + '/1474760145850806283/video/1' + } + ) + empty_resp = requests.packages.urllib3.response.HTTPResponse() + mock.head( + "https://twitter.com/BotPleroma/status/1323048312161947650" + "/photo/1", + status_code=200, + raw=empty_resp, + ) + mock.head( + "http://twitter.com/BotPleroma/status/1323048312161947650" + "/photo/1", + status_code=200, + raw=empty_resp, + ) + mock.head( + "https://twitter.com/BotPleroma/status/111242346465757545" + "/video/1", + status_code=200, + raw=empty_resp, + ) + mock.head( + "https://twitter.com/BotPleroma/status/" + "111242346465757545/video/10", + status_code=200, + raw=empty_resp, + ) mock.head("http://github.com", raw=empty_resp, status_code=200) mock.get(f"{twitter_base_url}/statuses/show.json", json=sample_data['tweet'], @@ -77,6 +120,38 @@ def _sample_users(mock_request, rootdir): for user_item in config_users['user_dict']: pinned = test_user.pinned pinned2 = test_user.pinned_2 + + mock.get("https://api.twitter.com/2/tweets/1339829031147954177" + "?poll.fields=duration_minutes%2Cend_datetime%2Cid" + "%2Coptions%2Cvoting_status&media.fields=duration_ms" + "%2Cheight%2Cmedia_key%2Cpreview_image_url%2Ctype%2Curl" + "%2Cwidth%2Cpublic_metrics&expansions=attachments" + ".poll_ids%2Cattachments.media_keys%2Cauthor_id" + "%2Centities.mentions.username%2Cgeo.place_id" + "%2Cin_reply_to_user_id%2Creferenced_tweets.id" + "%2Creferenced_tweets.id.author_id&tweet.fields" + "=attachments%2Cauthor_id%2Ccontext_annotations" + "%2Cconversation_id%2Ccreated_at%2Centities%2Cgeo%2Cid" + "%2Cin_reply_to_user_id%2Clang%2Cpublic_metrics" + "%2Cpossibly_sensitive%2Creferenced_tweets%2Csource" + "%2Ctext%2Cwithheld", + json=mock_request['sample_data']['pinned_tweet'], + status_code=200) + mock.get(f"{test_user.twitter_base_url_v2}/tweets/{pinned}" + f"?poll.fields=duration_minutes%2Cend_datetime%2Cid%2C" + f"options%2Cvoting_status&media.fields=duration_ms%2C" + f"height%2Cmedia_key%2Cpreview_image_url%2Ctype%2Curl%2C" + f"width%2Cpublic_metrics&expansions=attachments.poll_ids" + f"%2Cattachments.media_keys%2Cauthor_id%2C" + f"entities.mentions.username%2Cgeo.place_id%2C" + f"in_reply_to_user_id%2Creferenced_tweets.id%2C" + f"referenced_tweets.id.author_id&tweet.fields=attachments" + f"%2Cauthor_id%2Ccontext_annotations%2Cconversation_id%2" + f"Ccreated_at%2Centities%2Cgeo%2Cid%2Cin_reply_to_user_id" + f"%2Clang%2Cpublic_metrics%2Cpossibly_sensitive%2C" + f"referenced_tweets%2Csource%2Ctext%2Cwithheld", + json=mock_request['sample_data']['pinned_tweet'], + status_code=200) mock.get(f"{test_user.twitter_base_url_v2}/tweets/{pinned}" f"?poll.fields=duration_minutes%2Cend_datetime%2Cid%2C" f"options%2Cvoting_status&media.fields=duration_ms%2C" @@ -107,9 +182,10 @@ def _sample_users(mock_request, rootdir): f"referenced_tweets%2Csource%2Ctext%2Cwithheld", json=mock_request['sample_data']['pinned_tweet_2'], status_code=200) + user_v2 = f"user_v2_{user_item['twitter_username']}" mock.get(f"{twitter_base_url_v2}/users/by/username/" f"{user_item['twitter_username']}", - json=mock_request['sample_data']['pinned'], + json=mock_request['sample_data'][user_v2], status_code=200) mock.get(f"{twitter_base_url_v2}/users/by?" f"usernames={user_item['twitter_username']}", @@ -118,7 +194,7 @@ def _sample_users(mock_request, rootdir): mock.get(f"{test_user.twitter_base_url_v2}/users/by/username/" f"{user_item['twitter_username']}", - json=mock_request['sample_data']['pinned'], + json=mock_request['sample_data'][user_v2], status_code=200) mock.get(f"{test_user.twitter_base_url}/users/" @@ -218,6 +294,11 @@ def _sample_users(mock_request, rootdir): content=gif_content, headers={'Content-Type': 'image/gif'}, status_code=200) + mock.get("https://twitter.com/BotPleroma/status" + "/1323048312161947650/photo/1", + content=png_content, + headers={'Content-Type': 'image/png'}, + status_code=200) mock.get("https://pbs.twimg.com/media/ElxpP0hXEAI9X-H.jpg", content=png_content, headers={'Content-Type': 'image/png'}, @@ -226,6 +307,11 @@ def _sample_users(mock_request, rootdir): f"id=1323049214134407171", json=mock_request['sample_data']['tweet_video'], status_code=200) + mock.get("https://twitter.com/BotPleroma/status" + "/1474760145850806283/video/1", + content=mp4_content, + headers={'Content-Type': 'video/mp4'}, + status_code=200) mock.get("https://video.twimg.com/ext_tw_video/1323049175848833033" "/pu/vid/1280x720/de6uahiosn3VXMZO.mp4?tag=10", content=mp4_content, @@ -241,6 +327,9 @@ def _sample_users(mock_request, rootdir): profile_img_big = re.sub( r"normal", "400x400", profile_pic_url ) + mock.get(twitter_info["profile_image_url"], + content=profile_image_content, + status_code=200) mock.get(f"{profile_img_big}", content=profile_image_content, status_code=200) diff --git a/pleroma_bot/tests/test_exceptions.py b/pleroma_bot/tests/test_exceptions.py index 3542754..90af8d7 100644 --- a/pleroma_bot/tests/test_exceptions.py +++ b/pleroma_bot/tests/test_exceptions.py @@ -1,18 +1,22 @@ -import logging import os import re -import shutil import sys -from unittest.mock import patch +import time import pytest +import shutil import requests +import logging +import urllib.parse +from unittest.mock import patch from test_user import UserTemplate from conftest import get_config_users from pleroma_bot import cli from pleroma_bot.cli import User +from pleroma_bot._utils import Locker +from pleroma_bot._error import TimeoutLocker def test_user_invalid_pleroma_base(mock_request): @@ -59,16 +63,35 @@ def test_user_nitter_global(sample_users): with sample_user['mock'] as mock: config_users = get_config_users('config_nitter_global.yml') for user_item in config_users['user_dict']: - user_obj = User(user_item, config_users['config'], os.getcwd()) - nitter_url = f"https://nitter.net/{user_obj.twitter_username}" - assert user_obj.twitter_url is not None - assert user_obj.twitter_url == nitter_url + t_users = user_item["twitter_username"] + t_users_list = isinstance(t_users, list) + t_users = t_users if t_users_list else [t_users] + for t_user in t_users: + user_obj = User( + user_item, + config_users['config'], + os.getcwd() + ) + idx = user_obj.twitter_username.index(t_user) + nitter_url = f"https://nitter.net/" \ + f"{user_obj.twitter_username[idx]}" + assert user_obj.twitter_url[t_user] is not None + assert user_obj.twitter_url[t_user] == nitter_url config_users = get_config_users('config_nonitter.yml') # No global for user_item in config_users['user_dict']: - user_obj = User(user_item, config_users['config'], os.getcwd()) - twitter_url = f"http://twitter.com/{user_obj.twitter_username}" - assert user_obj.twitter_url == twitter_url + t_users = user_item["twitter_username"] + t_users_list = isinstance(t_users, list) + t_users = t_users if t_users_list else [t_users] + for t_user in t_users: + user_obj = User( + user_item, + config_users['config'], + os.getcwd() + ) + twitter_url = f"http://twitter.com/" \ + f"{user_obj.twitter_username[idx]}" + assert user_obj.twitter_url[t_user] == twitter_url return mock @@ -98,7 +121,7 @@ def test_user_invalid_max_tweets(sample_users): Check that an improper max_tweets value in the config raises a ValueError exception """ - error_str = 'max_tweets must be between 10 and 100. max_tweets: 5' + error_str = 'max_tweets must be between 10 and 3200. max_tweets: 5' with pytest.raises(ValueError) as error_info: for sample_user in sample_users: with sample_user['mock'] as mock: @@ -149,33 +172,38 @@ def test_check_pinned_exception_user(sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] + pinned = sample_user_obj.pinned_tweet_id + HTTPError = requests.exceptions.HTTPError + t_users = sample_user_obj.twitter_username + t_users_list = isinstance(t_users, list) + t_users = t_users if t_users_list else [t_users] + mock.get(url_user, json=mock_request['sample_data']['pinned_tweet'], status_code=500) - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, - sample_user_obj.pinned_tweet_id - ) - os.makedirs(tweet_folder, exist_ok=True) - HTTPError = requests.exceptions.HTTPError - with pytest.raises(HTTPError) as error_info: - sample_user_obj.check_pinned() + mock.get(f"{test_user.twitter_base_url_v2}/tweets?ids={pinned}" + f"&expansions=attachments.poll_ids" + f"&poll.fields=duration_minutes%2Coptions", + json=mock_request['sample_data']['poll'], + status_code=200) - exception_value = ( - f"500 Server Error: None for url: {url_user}" - ) - assert str(error_info.value) == exception_value - pin_p = os.path.join( - sample_user_obj.user_path, "pinned_id_pleroma.txt" - ) - pin_t = os.path.join( - sample_user_obj.user_path, "pinned_id.txt" - ) - if os.path.isfile(pin_p): - os.remove(pin_p) - if os.path.isfile(pin_t): - os.remove(pin_t) - os.rmdir(tweet_folder) + for t_user in t_users: + with pytest.raises(HTTPError) as error_info: + sample_user_obj.check_pinned() + exception_value = ( + f"500 Server Error: None for url: {url_user}" + ) + assert str(error_info.value) == exception_value + pin_p = os.path.join( + sample_user_obj.user_path[t_user], "pinned_id_pleroma.txt" + ) + pin_t = os.path.join( + sample_user_obj.user_path[t_user], "pinned_id.txt" + ) + if os.path.isfile(pin_p): + os.remove(pin_p) + if os.path.isfile(pin_t): + os.remove(pin_t) def test_check_pinned_exception_tweet(sample_users, mock_request): @@ -199,12 +227,19 @@ def test_check_pinned_exception_tweet(sample_users, mock_request): f"500 Server Error: None for url: {url_tweet}" ) assert str(error_info.value) == exception_value - pin_p = os.path.join(sample_user_obj.user_path, "pinned_id_pleroma.txt") - pin_t = os.path.join(sample_user_obj.user_path, "pinned_id.txt") - if os.path.isfile(pin_p): - os.remove(pin_p) - if os.path.isfile(pin_t): - os.remove(pin_t) + for t_user in sample_user_obj.twitter_username: + pin_p = os.path.join( + sample_user_obj.user_path[t_user], + "pinned_id_pleroma.txt" + ) + pin_t = os.path.join( + sample_user_obj.user_path[t_user], + "pinned_id.txt" + ) + if os.path.isfile(pin_p): + os.remove(pin_p) + if os.path.isfile(pin_t): + os.remove(pin_t) def test_pin_pleroma_exception(sample_users, mock_request): @@ -218,8 +253,12 @@ def test_pin_pleroma_exception(sample_users, mock_request): status_code=500) pin_id = sample_user_obj.pin_pleroma(test_user.pleroma_pinned_new) assert pin_id is None - pin_p = os.path.join(sample_user_obj.user_path, "pinned_id_pleroma.txt") - os.remove(pin_p) + for t_user in sample_user_obj.twitter_username: + pin_p = os.path.join( + sample_user_obj.user_path[t_user], + "pinned_id_pleroma.txt" + ) + os.remove(pin_p) def test_unpin_pleroma_exception(sample_users, mock_request): @@ -233,16 +272,18 @@ def test_unpin_pleroma_exception(sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - mock.post(url_unpin, - json={}, - status_code=500) - pinned_file = os.path.join( - sample_user_obj.user_path, "pinned_id_pleroma.txt" - ) - with open(pinned_file, 'w') as file: - file.write(test_user.pleroma_pinned) - file.close() - sample_user_obj.unpin_pleroma(pinned_file) + for t_user in sample_user_obj.twitter_username: + mock.post(url_unpin, + json={}, + status_code=500) + pinned_file = os.path.join( + sample_user_obj.user_path[t_user], + "pinned_id_pleroma.txt" + ) + with open(pinned_file, 'w') as file: + file.write(test_user.pleroma_pinned) + file.close() + sample_user_obj.unpin_pleroma(pinned_file) exception_value = ( f"500 Server Error: None for url: {url_unpin}" @@ -291,19 +332,23 @@ def test_unpin_pleroma_statuses_exception(sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - url_statuses = ( - f"{test_user.pleroma_base_url}" - f"/api/v1/accounts/" - f"{sample_user_obj.pleroma_username}/statuses" - ) - mock.get( - url_statuses, - json=mock_request['sample_data']['pleroma_statuses_pin'], - status_code=500 - ) - pinned_file = os.path.join(sample_user_obj.user_path, - "pinned_id_pleroma.txt") - sample_user_obj.unpin_pleroma(pinned_file) + for t_user in sample_user_obj.twitter_username: + url_statuses = ( + f"{test_user.pleroma_base_url}" + f"/api/v1/accounts/" + f"{sample_user_obj.pleroma_username}/statuses" + ) + sample_data = mock_request['sample_data'] + mock.get( + url_statuses, + json=sample_data['pleroma_statuses_pin'], + status_code=500 + ) + pinned_file = os.path.join( + sample_user_obj.user_path[t_user], + "pinned_id_pleroma.txt" + ) + sample_user_obj.unpin_pleroma(pinned_file) exception_value = f"500 Server Error: None for url: {url_statuses}" assert str(error_info.value) == exception_value @@ -314,21 +359,24 @@ def test__get_pinned_tweet_id_exception(sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - pinned = sample_user_obj.pinned_tweet_id - assert pinned == test_user.pinned - pinned_url = ( - f"{test_user.twitter_base_url_v2}/users/by/username/" - f"{sample_user_obj.twitter_username}?user.fields=" - f"pinned_tweet_id&expansions=pinned_tweet_id&" - f"tweet.fields=entities" - ) - mock.get(pinned_url, - json=mock_request['sample_data']['pinned'], - status_code=500) - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj._get_pinned_tweet_id() - exception_value = f"500 Server Error: None for url: {pinned_url}" - assert str(error_info.value) == exception_value + for t_user in sample_user_obj.twitter_username: + pinned = sample_user_obj.pinned_tweet_id + assert pinned == test_user.pinned + pinned_url = ( + f"{test_user.twitter_base_url_v2}/users/by/username/" + f"{t_user}?user.fields=" + f"pinned_tweet_id&expansions=pinned_tweet_id&" + f"tweet.fields=entities" + ) + mock.get(pinned_url, + json=mock_request['sample_data']['pinned'], + status_code=500) + err_ex = requests.exceptions.HTTPError + with pytest.raises(err_ex) as error_info: + sample_user_obj._get_pinned_tweet_id() + exception_value = f"500 Server Error: " \ + f"None for url: {pinned_url}" + assert str(error_info.value) == exception_value def test_post_pleroma_exception(sample_users, mock_request): @@ -336,18 +384,20 @@ def test_post_pleroma_exception(sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - tweets_folder = sample_user_obj.tweets_temp_path - tweet_folder = os.path.join(tweets_folder, test_user.pinned) - os.makedirs(tweet_folder, exist_ok=True) - post_url = f"{test_user.pleroma_base_url}/api/v1/statuses" - mock.post(post_url, status_code=500) - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj.post_pleroma( - (test_user.pinned, "", ""), None, False - ) - exception_value = f"500 Server Error: None for url: {post_url}" - assert str(error_info.value) == exception_value - os.rmdir(tweet_folder) + for t_user in sample_user_obj.twitter_username: + tweets_folder = sample_user_obj.tweets_temp_path + tweet_folder = os.path.join(tweets_folder, test_user.pinned) + os.makedirs(tweet_folder, exist_ok=True) + post_url = f"{test_user.pleroma_base_url}/api/v1/statuses" + mock.post(post_url, status_code=500) + err_ex = requests.exceptions.HTTPError + with pytest.raises(err_ex) as error_info: + sample_user_obj.post_pleroma( + (test_user.pinned, "", ""), None, False + ) + exception_value = f"500 Server Error: None for url: {post_url}" + assert str(error_info.value) == exception_value + os.rmdir(tweet_folder) def test_update_pleroma_exception(rootdir, mock_request, sample_users, caplog): @@ -374,102 +424,114 @@ def test_update_pleroma_exception(rootdir, mock_request, sample_users, caplog): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - mock.get(profile_url, - content=profile_image_content, - status_code=500) - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj.update_pleroma() - exception_value = f"500 Server Error: None for url: {profile_url}" - assert str(error_info.value) == exception_value - mock.get(profile_url, - content=profile_image_content, - status_code=200) - mock.get(banner_url, - content=profile_banner_content, - status_code=500) - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj.update_pleroma() - exception_value = f"500 Server Error: None for url: {banner_url}" - assert str(error_info.value) == exception_value - cred_url = ( - f"{test_user.pleroma_base_url}/api/v1/" - f"accounts/update_credentials" - ) - mock.patch(cred_url, - status_code=500) - mock.get(profile_url, - content=profile_image_content, - status_code=200) - mock.get(banner_url, - content=profile_banner_content, - status_code=200) - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj.update_pleroma() - exception_value = f"500 Server Error: None for url: {cred_url}" - assert str(error_info.value) == exception_value - mock.patch(cred_url, - status_code=422) - mock.get(profile_url, - content=profile_image_content, - status_code=200) - mock.get(banner_url, - content=profile_banner_content, - status_code=200) - with caplog.at_level(logging.ERROR): - sample_user_obj.update_pleroma() + for t_user in sample_user_obj.twitter_username: + mock.get(profile_url, + content=profile_image_content, + status_code=500) + err_ex = requests.exceptions.HTTPError + with pytest.raises(err_ex) as error_info: + sample_user_obj.update_pleroma() + exception_value = f"500 Server Error: None " \ + f"for url: {profile_url}" + assert str(error_info.value) == exception_value + mock.get(profile_url, + content=profile_image_content, + status_code=200) + mock.get(banner_url, + content=profile_banner_content, + status_code=500) + with pytest.raises(err_ex) as error_info: + sample_user_obj.update_pleroma() + exception_value = f"500 Server Error: " \ + f"None for url: {banner_url}" + assert str(error_info.value) == exception_value + cred_url = ( + f"{test_user.pleroma_base_url}/api/v1/" + f"accounts/update_credentials" + ) + mock.patch(cred_url, + status_code=500) + mock.get(profile_url, + content=profile_image_content, + status_code=200) + mock.get(banner_url, + content=profile_banner_content, + status_code=200) + with pytest.raises(err_ex) as error_info: + sample_user_obj.update_pleroma() + exception_value = f"500 Server Error: None for url: {cred_url}" + assert str(error_info.value) == exception_value + mock.patch(cred_url, + status_code=422) + mock.get(profile_url, + content=profile_image_content, + status_code=200) + mock.get(banner_url, + content=profile_banner_content, + status_code=200) + with caplog.at_level(logging.ERROR): + sample_user_obj.update_pleroma() + exception_value = ( + "Exception occurred" + "\nError code 422" + "\n(Unprocessable Entity)" + "\nPlease check that the bio text or " + "the metadata fields text" + "\naren't too long." + ) + assert exception_value in caplog.text + mock_fields = [ + {'name': 'Field1', 'value': 'Value1'}, + {'name': 'Field2', 'value': 'Value2'}, + {'name': 'Field3', 'value': 'Value3'}, + {'name': 'Field4', 'value': 'Value4'}, + {'name': 'Field5', 'value': 'Value5'} + ] + sample_user_obj.fields = mock_fields + with pytest.raises(Exception) as error_info: + sample_user_obj.update_pleroma() exception_value = ( - "Exception occurred" - "\nError code 422" - "\n(Unprocessable Entity)" - "\nPlease check that the bio text or " - "the metadata fields text" - "\naren't too long." + f"Total number of metadata fields cannot " + f"exceed 4.\nProvided: {len(mock_fields)}. Exiting..." ) - assert exception_value in caplog.text - mock_fields = [ - {'name': 'Field1', 'value': 'Value1'}, - {'name': 'Field2', 'value': 'Value2'}, - {'name': 'Field3', 'value': 'Value3'}, - {'name': 'Field4', 'value': 'Value4'}, - {'name': 'Field5', 'value': 'Value5'} - ] - sample_user_obj.fields = mock_fields - with pytest.raises(Exception) as error_info: - sample_user_obj.update_pleroma() - exception_value = ( - f"Total number of metadata fields cannot " - f"exceed 4.\nProvided: {len(mock_fields)}. Exiting..." - ) - assert str(error_info.value) == exception_value + assert str(error_info.value) == exception_value def test__get_tweets_exception(sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - tweet_id_url = ( - f"{sample_user_obj.twitter_base_url}/statuses/" - f"show.json?id={str(sample_user_obj.pinned_tweet_id)}" - ) - mock.get(tweet_id_url, status_code=500) - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj._get_tweets( - "v1.1", sample_user_obj.pinned_tweet_id + for t_user in sample_user_obj.twitter_username: + idx = sample_user_obj.twitter_username.index(t_user) + tweet_id_url = ( + f"{sample_user_obj.twitter_base_url}" + f"/statuses/show.json?id=" + f"{str(sample_user_obj.pinned_tweet_id)}" ) - exception_value = f"500 Server Error: None for url: {tweet_id_url}" - assert str(error_info.value) == exception_value - tweets_url = ( - f"{sample_user_obj.twitter_base_url}" - f"/statuses/user_timeline.json?screen_name=" - f"{sample_user_obj.twitter_username}" - f"&count={str(sample_user_obj.max_tweets)}&include_rts=true" - ) - mock.get(tweets_url, status_code=500) - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj._get_tweets("v1.1") - exception_value = f"500 Server Error: None for url: {tweets_url}" - assert str(error_info.value) == exception_value + + mock.get(tweet_id_url, status_code=500) + err_ex = requests.exceptions.HTTPError + with pytest.raises(err_ex) as error_info: + sample_user_obj._get_tweets( + "v1.1", sample_user_obj.pinned_tweet_id + ) + exception_value = f"500 Server Error: " \ + f"None for url: {tweet_id_url}" + assert str(error_info.value) == exception_value + tweets_url = ( + f"{sample_user_obj.twitter_base_url}" + f"/statuses/user_timeline.json?screen_name=" + f"{sample_user_obj.twitter_username[idx]}" + f"&count={str(sample_user_obj.max_tweets)}" + f"&include_rts=true" + ) + mock.get(tweets_url, status_code=500) + with pytest.raises(err_ex) as error_info: + sample_user_obj._get_tweets("v1.1") + exception_value = f"500 Server Error: " \ + f"None for url: {tweets_url}" + assert str(error_info.value) == exception_value def test__get_tweets_v2_exception(sample_users): @@ -477,34 +539,106 @@ def test__get_tweets_v2_exception(sample_users): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - tweets_url = ( - f"{test_user.twitter_base_url_v2}/users/by?" - f"usernames={sample_user_obj.twitter_username}" - ) - mock.get(tweets_url, status_code=500) - start_time = sample_user_obj.get_date_last_pleroma_post() - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj._get_tweets( - "v2", start_time=start_time + for t_user in sample_user_obj.twitter_username: + idx = sample_user_obj.twitter_username.index(t_user) + date = sample_user_obj.get_date_last_pleroma_post() + date_encoded = urllib.parse.quote(date) + tweets_url = ( + f"{test_user.twitter_base_url_v2}/users/2244994945/tweets" + f"?max_results={sample_user_obj.max_tweets}&start_time" + f"={date_encoded}&poll." + f"fields=duration_minutes%2Cend_datetime%2Cid" + f"%2Coptions%2Cvoting_status&media.fields=duration_ms" + f"%2Cheight%2Cmedia_key%2Cpreview_image_url%2Ctype" + f"%2Curl%2Cwidth%2Cpublic_metrics&expansions=" + f"attachments.poll_ids%2Cattachments.media_keys" + f"%2Cauthor_id%2Centities.mentions.username" + f"%2Cgeo.place_id%2Cin_reply_to_user_id%2C" + f"referenced_tweets.id%2Creferenced_tweets.id." + f"author_id&tweet.fields=attachments%2Cauthor_id" + f"%2Ccontext_annotations%2Cconversation_id%2C" + f"created_at%2Centities%2Cgeo%2Cid%2C" + f"in_reply_to_user_id%2Clang%2Cpublic_metrics%2C" + f"possibly_sensitive%2Creferenced_tweets%2Csource%2C" + f"text%2Cwithheld" ) - exception_value = f"500 Server Error: None for url: {tweets_url}" - assert str(error_info.value) == exception_value + count = sample_user_obj.max_tweets + 100 + start_time = sample_user_obj.get_date_last_pleroma_post() + sample_user_obj._get_tweets_v2( + start_time=start_time, t_user=t_user, count=count + ) + mock.get(tweets_url, status_code=500) + + err_ex = requests.exceptions.HTTPError + with pytest.raises(err_ex) as error_info: + sample_user_obj._get_tweets( + "v2", start_time=start_time, t_user=t_user + ) + + exception_value = f"500 Server Error: " \ + f"None for url: {tweets_url}" + assert str(error_info.value) == exception_value + + tweets_url = ( + f"{test_user.twitter_base_url_v2}/users/by?" + f"usernames={sample_user_obj.twitter_username[idx]}" + ) + mock.get(tweets_url, status_code=500) + start_time = sample_user_obj.get_date_last_pleroma_post() + err_ex = requests.exceptions.HTTPError + with pytest.raises(err_ex) as error_info: + sample_user_obj._get_tweets( + "v2", start_time=start_time, t_user=t_user + ) + exception_value = f"500 Server Error: " \ + f"None for url: {tweets_url}" + assert str(error_info.value) == exception_value def test__get_twitter_info_exception(sample_users): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - info_url = ( - f"{sample_user_obj.twitter_base_url}" - f"/users/show.json?screen_name=" - f"{sample_user_obj.twitter_username}" - ) - mock.get(info_url, status_code=500) - with pytest.raises(requests.exceptions.HTTPError) as error_info: - sample_user_obj._get_twitter_info() - exception_value = f"500 Server Error: None for url: {info_url}" - assert str(error_info.value) == exception_value + t_users = sample_user_obj.twitter_username + t_users_list = isinstance(t_users, list) + t_users = t_users if t_users_list else [t_users] + for t_user in t_users: + idx = sample_user_obj.twitter_username.index(t_user) + info_url = ( + f"{sample_user_obj.twitter_base_url}" + f"/users/show.json?screen_name=" + f"{sample_user_obj.twitter_username[idx]}" + ) + mock.get(info_url, status_code=500) + err_ex = requests.exceptions.HTTPError + with pytest.raises(err_ex) as error_info: + sample_user_obj._get_twitter_info() + exception_value = f"500 Server Error: None for url: {info_url}" + assert str(error_info.value) == exception_value + + url_username = ( + f"https://api.twitter.com/2/users/by/username/" + f"{t_user}?user.fields=created_at%2Cdescripti" + f"on%2Centities%2Cid%2Clocation%2Cname%2Cpinn" + f"ed_tweet_id%2Cprofile_image_url%2Cprotecte" + f"d%2Curl%2Cusername%2Cverified%2Cwithhe" + f"ld&expansions=pinned_tweet_id&tweet." + f"fields=attachments%2Cauthor_id%2C" + f"context_annotations%2Cconversatio" + f"n_id%2Ccreated_at%2Centities%2Cgeo%2Cid%2Cin" + f"_reply_to_user_id%2Clang%2Cpublic_metrics" + f"%2Cpossibly_sensitive%2C" + f"referenced_tweets%2Csource%2Ctext%2Cwithheld" + ) + mock.get(f"{sample_user_obj.twitter_base_url_v2}/users/by/" + f"username/{t_user}", + status_code=500) + with pytest.raises(err_ex) as error_info: + sample_user_obj._get_twitter_info() + exception_value = ( + f"500 Server Error: None for url: {url_username}" + ) + assert str(error_info.value) == exception_value def test_main_oauth_exception( @@ -532,7 +666,7 @@ def test_main_oauth_exception( monkeypatch.setattr('builtins.input', lambda: "2020-12-30") with patch.object(sys, 'argv', ['']): with caplog.at_level(logging.ERROR): - assert cli.main() == 1 + assert cli.main() == 0 err_msg = ( "Unable to retrieve tweets. Is the account protected? " "If so, you need to provide the following OAuth 1.0a " @@ -549,18 +683,24 @@ def test_main_oauth_exception( shutil.copy(backup_config, prev_config) for sample_user in sample_users: sample_user_obj = sample_user['user_obj'] - pinned_path = os.path.join(os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id.txt') - pinned_pleroma = os.path.join(os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id_pleroma.txt') - if os.path.isfile(pinned_path): - os.remove(pinned_path) - if os.path.isfile(pinned_pleroma): - os.remove(pinned_pleroma) + for t_user in sample_user_obj.twitter_username: + idx = sample_user_obj.twitter_username.index(t_user) + pinned_path = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id.txt' + ) + pinned_pleroma = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id_pleroma.txt' + ) + if os.path.isfile(pinned_path): + os.remove(pinned_path) + if os.path.isfile(pinned_pleroma): + os.remove(pinned_pleroma) return g_mock @@ -615,8 +755,20 @@ def test__expand_urls(sample_users, mock_request): sample_user_obj = sample_user['user_obj'] fake_url = "https://cutt.ly/xg3TuY0" mock.head(fake_url, status_code=500) - tweet = mock_request['sample_data']['pinned_tweet']['data'] + tweet = mock_request['sample_data']['pinned_tweet_3']['data'] with pytest.raises(requests.exceptions.HTTPError) as error_info: sample_user_obj._expand_urls(tweet) exception_value = f"500 Server Error: None for url: {fake_url}" assert str(error_info.value) == exception_value + + +def test_locker(): + with pytest.raises(TimeoutLocker) as error_info: + with Locker(): + with Locker(): + time.sleep(20) + exception_value = ( + "The file lock '/tmp/pleroma_bot.lock' could not be acquired. Is " + "another instance of pleroma-bot running?" + ) + assert str(error_info.value) == exception_value diff --git a/pleroma_bot/tests/test_files/config_invidious_different_instance.yml b/pleroma_bot/tests/test_files/config_invidious_different_instance.yml new file mode 100644 index 0000000..4239ede --- /dev/null +++ b/pleroma_bot/tests/test_files/config_invidious_different_instance.yml @@ -0,0 +1,54 @@ +twitter_base_url: https://api.twitter.com/1.1 +# Change this to your Fediverse instance +pleroma_base_url: https://pleroma.robertoszek.xyz +# How many tweets to get in every execution +# Twitter's API hard limit is 3,200 +max_tweets: 40 +# Twitter bearer token +twitter_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +nitter: true +invidious_base_url: https://another.invidious.instance +# List of users and their attributes +users: +- twitter_username: KyleBosman + pleroma_username: KyleBosman + pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + # If you want to add a link to the original status or not + signature: true + # If you want to download Twitter attachments and add them to the Pleroma posts + media_upload: true + # If twitter links should be changed to nitter.net ones + # If mentions should be transformed to links to the mentioned Twitter profile + rich_text: true + invidious: true + invidious_base_url: https://test.invidious.instance + # visibility of the post. Must one of the following: public, unlisted, private, direct + visibility: "unlisted" + # Force all posts for this account to be sensitive or not + # The NSFW banner for the instance will be shown for attachments as a warning if true + # If not defined, the original tweet sensitivity will be used on a tweet by tweet basis + sensitive: false + support_account: robertoszek + # you can use any attribute from 'user' inside a string with {{ attr_name }} and it will be replaced + # with the attribute value. e.g. {{ support_account }} + bio_text: "\U0001F916 BEEP BOOP \U0001F916 \nI'm a bot that mirrors {{ twitter_username }} Twitter's\ + \ account. \nAny issues please contact @{{ support_account }} \n \n " # username will be replaced by its value + # Optional metadata fields and values for the Pleroma profile + fields: + - name: "\U0001F426 Birdsite" + value: "{{ twitter_url }}" + - name: "Status" + value: "I am completely operational, and all my circuits are functioning perfectly." + - name: "Source" + value: "https://gitea.robertoszek.xyz/robertoszek/pleroma-twitter-info-grabber" +- twitter_username: arstechnica + pleroma_username: mynewsbot + pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + signature: true + media_upload: false + pleroma_url: https://another.pleroma.instance + max_tweets: 50 + invidious: true + invidious_base_url: https://another.invidious.instance + bio_text: "\U0001F916 BEEP BOOP \U0001F916 \n I'm a bot that mirrors {{ twitter_username }} Twitter's\ + \ account. \n Any issues please contact @robertoszek \n \n " diff --git a/pleroma_bot/tests/test_files/config_keep_media_links.yml b/pleroma_bot/tests/test_files/config_keep_media_links.yml new file mode 100644 index 0000000..8a88101 --- /dev/null +++ b/pleroma_bot/tests/test_files/config_keep_media_links.yml @@ -0,0 +1,51 @@ +twitter_base_url: https://api.twitter.com/1.1 +# Change this to your Fediverse instance +pleroma_base_url: https://pleroma.robertoszek.xyz +# How many tweets to get in every execution +# Twitter's API hard limit is 3,200 +max_tweets: 40 +# Twitter bearer token +twitter_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# List of users and their attributes +users: +- twitter_username: KyleBosman + pleroma_username: KyleBosman + pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + # If you want to add a link to the original status or not + signature: true + # If you want to download Twitter attachments and add them to the Pleroma posts + media_upload: true + keep_media_links: true + # If twitter links should be changed to nitter.net ones + # If mentions should be transformed to links to the mentioned Twitter profile + rich_text: true + # visibility of the post. Must one of the following: public, unlisted, private, direct + visibility: "unlisted" + # Force all posts for this account to be sensitive or not + # The NSFW banner for the instance will be shown for attachments as a warning if true + # If not defined, the original tweet sensitivity will be used on a tweet by tweet basis + sensitive: false + support_account: robertoszek + # you can use any attribute from 'user' inside a string with {{ attr_name }} and it will be replaced + # with the attribute value. e.g. {{ support_account }} + bio_text: "\U0001F916 BEEP BOOP \U0001F916 \nI'm a bot that mirrors {{ twitter_username }} Twitter's\ + \ account. \nAny issues please contact @{{ support_account }} \n \n " # username will be replaced by its value + # Optional metadata fields and values for the Pleroma profile + fields: + - name: "\U0001F426 Birdsite" + value: "{{ twitter_url }}" + - name: "Status" + value: "I am completely operational, and all my circuits are functioning perfectly." + - name: "Source" + value: "https://gitea.robertoszek.xyz/robertoszek/pleroma-twitter-info-grabber" +- twitter_username: arstechnica + pleroma_username: mynewsbot + pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + signature: true + original_date_format: "%Y/%m/%d %H:%M" + media_upload: false + pleroma_url: https://another.pleroma.instance + keep_media_links: true + max_tweets: 50 + bio_text: "\U0001F916 BEEP BOOP \U0001F916 \n I'm a bot that mirrors {{ twitter_username }} Twitter's\ + \ account. \n Any issues please contact @robertoszek \n \n " diff --git a/pleroma_bot/tests/test_files/config_no_keep_media_links.yml b/pleroma_bot/tests/test_files/config_no_keep_media_links.yml new file mode 100644 index 0000000..b870f69 --- /dev/null +++ b/pleroma_bot/tests/test_files/config_no_keep_media_links.yml @@ -0,0 +1,49 @@ +twitter_base_url: https://api.twitter.com/1.1 +# Change this to your Fediverse instance +pleroma_base_url: https://pleroma.robertoszek.xyz +# How many tweets to get in every execution +# Twitter's API hard limit is 3,200 +max_tweets: 40 +# Twitter bearer token +twitter_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX +# List of users and their attributes +users: +- twitter_username: KyleBosman + pleroma_username: KyleBosman + pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + # If you want to add a link to the original status or not + signature: true + # If you want to download Twitter attachments and add them to the Pleroma posts + media_upload: true + # If twitter links should be changed to nitter.net ones + # If mentions should be transformed to links to the mentioned Twitter profile + rich_text: true + # visibility of the post. Must one of the following: public, unlisted, private, direct + visibility: "unlisted" + # Force all posts for this account to be sensitive or not + # The NSFW banner for the instance will be shown for attachments as a warning if true + # If not defined, the original tweet sensitivity will be used on a tweet by tweet basis + sensitive: false + support_account: robertoszek + # you can use any attribute from 'user' inside a string with {{ attr_name }} and it will be replaced + # with the attribute value. e.g. {{ support_account }} + bio_text: "\U0001F916 BEEP BOOP \U0001F916 \nI'm a bot that mirrors {{ twitter_username }} Twitter's\ + \ account. \nAny issues please contact @{{ support_account }} \n \n " # username will be replaced by its value + # Optional metadata fields and values for the Pleroma profile + fields: + - name: "\U0001F426 Birdsite" + value: "{{ twitter_url }}" + - name: "Status" + value: "I am completely operational, and all my circuits are functioning perfectly." + - name: "Source" + value: "https://gitea.robertoszek.xyz/robertoszek/pleroma-twitter-info-grabber" +- twitter_username: arstechnica + pleroma_username: mynewsbot + pleroma_token: XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX + signature: true + original_date_format: "%Y/%m/%d %H:%M" + media_upload: false + pleroma_url: https://another.pleroma.instance + max_tweets: 50 + bio_text: "\U0001F916 BEEP BOOP \U0001F916 \n I'm a bot that mirrors {{ twitter_username }} Twitter's\ + \ account. \n Any issues please contact @robertoszek \n \n " diff --git a/pleroma_bot/tests/test_files/sample_data/media/twitter-archive.zip b/pleroma_bot/tests/test_files/sample_data/media/twitter-archive.zip new file mode 100644 index 0000000..bdecf9b Binary files /dev/null and b/pleroma_bot/tests/test_files/sample_data/media/twitter-archive.zip differ diff --git a/pleroma_bot/tests/test_files/sample_data/pinned_tweet.json b/pleroma_bot/tests/test_files/sample_data/pinned_tweet.json index b0367d8..e12355c 100644 --- a/pleroma_bot/tests/test_files/sample_data/pinned_tweet.json +++ b/pleroma_bot/tests/test_files/sample_data/pinned_tweet.json @@ -6,7 +6,8 @@ "1323049466027479040" ] }, - "text": "Cutt.ly/xg3TuY0 Poll - @Mention Rogue link https://cutt.ly/xg3TuY0 and https://cutofflink.com/pathโ€ฆ", + "text": "Cutt.ly/xg3TuY0 https://twitter.com/BotPleroma/status/111242346465757545/video/10 Poll - @Mention Rogue link and https://cutofflink.com/pathโ€ฆ", + "public_metrics": { "retweet_count": 0, "reply_count": 0, diff --git a/pleroma_bot/tests/test_files/sample_data/pinned_tweet_2.json b/pleroma_bot/tests/test_files/sample_data/pinned_tweet_2.json index 8b7e693..0108513 100644 --- a/pleroma_bot/tests/test_files/sample_data/pinned_tweet_2.json +++ b/pleroma_bot/tests/test_files/sample_data/pinned_tweet_2.json @@ -6,7 +6,7 @@ "1323049466027479040" ] }, - "text": "Poll", + "text": "https://twitter.com/BotPleroma/status/1323048312161947650/photo/1 Poll https://cutt.ly/xg3TuY0 Yeyeye ", "public_metrics": { "retweet_count": 0, "reply_count": 0, diff --git a/pleroma_bot/tests/test_files/sample_data/pinned_tweet_3.json b/pleroma_bot/tests/test_files/sample_data/pinned_tweet_3.json new file mode 100644 index 0000000..906851d --- /dev/null +++ b/pleroma_bot/tests/test_files/sample_data/pinned_tweet_3.json @@ -0,0 +1,69 @@ +{ + "data": { + "created_at": "2020-11-01T23:49:08.000Z", + "attachments": { + "poll_ids": [ + "1323049466027479040" + ] + }, + "text": "https://cutt.ly/xg3TuY0 Github.com Poll https://twitter.com/BotPleroma/status/1323048312161947650/photo/1 Yeyeye", + "public_metrics": { + "retweet_count": 0, + "reply_count": 0, + "like_count": 0, + "quote_count": 0 + }, + "lang": "en", + "author_id": "1320506197913542656", + "source": "Twitter Web App", + "conversation_id": "1323049466837032961", + "possibly_sensitive": false, + "id": "1323049466837032962" + }, + "includes": { + "polls": [ + { + "id": "1323049466027479040", + "voting_status": "open", + "end_datetime": "2020-11-08T23:49:08.000Z", + "duration_minutes": 10080, + "options": [ + { + "position": 1, + "label": "๐Ÿ˜Š", + "votes": 0 + }, + { + "position": 2, + "label": "๐Ÿ˜Ÿ", + "votes": 0 + }, + { + "position": 3, + "label": "๐Ÿค”", + "votes": 0 + } + ] + } + ], + "media": [ + { + "preview_image_url": "https://pbs.twimg.com/ext_tw_video_thumb/1323049175848833033/pu/img/_xRqkGnMX3DEB4UM.jpg", + "type": "video", + "media_key": "7_1323049175848833033", + "duration_ms": 16684, + "height": 720, + "width": 1280, + "public_metrics": { + "view_count": 0 + } + }], + "users": [ + { + "id": "1320506197913542656", + "name": "test-pleroma-bot", + "username": "BotPleroma" + } + ] + } +} \ No newline at end of file diff --git a/pleroma_bot/tests/test_files/sample_data/tweets_v2.json b/pleroma_bot/tests/test_files/sample_data/tweets_v2.json index 6d6e01d..e2e0f23 100644 --- a/pleroma_bot/tests/test_files/sample_data/tweets_v2.json +++ b/pleroma_bot/tests/test_files/sample_data/tweets_v2.json @@ -1,5 +1,38 @@ { "data": [ + { + "in_reply_to_user_id": "866105538", + "referenced_tweets": [ + { + "type": "replied_to", + "id": "1347782491248054272" + } + ], + "author_id": "1320506197913542656", + "lang": "en", + "possibly_sensitive": false, + "public_metrics": { + "retweet_count": 0, + "reply_count": 1, + "like_count": 37, + "quote_count": 0 + }, + "text": "@imdevKc It's not hypothetical. It's based on data. #sponsored", + "entities": { + "mentions": [ + { + "start": 0, + "end": 8, + "username": "imdevKc" + } + ], + "hashtags": [{"start": 52, "end": 62, "tag": "notsponsored"}, {"start": 52, "end": 62, "tag": "sponsored"}] + }, + "conversation_id": "1347717302280712192", + "created_at": "2021-01-09T05:51:31.000Z", + "source": "Twitter Web App", + "id": "1347783039334670336" + }, { "created_at": "2020-11-01T23:49:08.000Z", "attachments": { @@ -7,7 +40,7 @@ "1323049466027479040" ] }, - "text": "Poll", + "text": "Github.com Poll", "public_metrics": { "retweet_count": 0, "reply_count": 0, @@ -40,7 +73,7 @@ ], "hashtags": [{"start": 52, "end": 62, "tag": "notsponsored"}] }, - "text": "RT Video https://t.co/yMAZ4i0t0W", + "text": "Github.com RT Video https://t.co/yMAZ4i0t0W https://twitter.com/BotPleroma/status/1323049214134407171/video/1", "public_metrics": { "retweet_count": 0, "reply_count": 0, @@ -78,7 +111,7 @@ } ] }, - "text": "Animated GIF https://t.co/VzXVzMNIMF", + "text": "Animated GIF https://twitter.com/BotPleroma/status/1323048312161947650/photo/1", "public_metrics": { "retweet_count": 0, "reply_count": 0, @@ -124,7 +157,7 @@ ], "hashtags": [{"start": 30, "end": 40, "tag": "sponsored"}] }, - "text": "Image https://t.co/paDoCjOmxr #sponsored", + "text": "Image https://t.co/paDoCjOmxr #sponsored https://youtube.com/watch?v=dQw4w9WgXcQ", "public_metrics": { "retweet_count": 0, "reply_count": 0, @@ -137,40 +170,8 @@ "conversation_id": "1323048139251658753", "possibly_sensitive": false, "id": "1323048139251658753" - }, - { - "in_reply_to_user_id": "866105538", - "referenced_tweets": [ - { - "type": "replied_to", - "id": "1347782491248054272" - } - ], - "author_id": "81290806", - "lang": "en", - "possibly_sensitive": false, - "public_metrics": { - "retweet_count": 0, - "reply_count": 1, - "like_count": 37, - "quote_count": 0 - }, - "text": "@imdevKc It's not hypothetical. It's based on data. #sponsored", - "entities": { - "mentions": [ - { - "start": 0, - "end": 8, - "username": "imdevKc" - } - ], - "hashtags": [{"start": 52, "end": 62, "tag": "notsponsored"}, {"start": 52, "end": 62, "tag": "sponsored"}] - }, - "conversation_id": "1347717302280712192", - "created_at": "2021-01-09T05:51:31.000Z", - "source": "Twitter Web App", - "id": "1347783039334670336" } + ], "includes": { "polls": [ @@ -198,13 +199,6 @@ ] } ], - "users": [ - { - "id": "1320506197913542656", - "name": "test-pleroma-bot", - "username": "BotPleroma" - } - ], "media": [ { "preview_image_url": "https://pbs.twimg.com/ext_tw_video_thumb/1323049175848833033/pu/img/_xRqkGnMX3DEB4UM.jpg", @@ -232,11 +226,17 @@ "width": 960 } ], - "tweets": [] + "tweets": [{ + "type": "photo", + "media_key": "3_1323048111057604610", + "url": "https://pbs.twimg.com/media/ElxpP0hXEAI9X-H.jpg", + "height": 540, + "width": 960 + }] }, "meta": { "newest_id": "1323049466837032961", "oldest_id": "1323048139251658753", - "result_count": 4 + "result_count": 5 } } diff --git a/pleroma_bot/tests/test_files/sample_data/tweets_v2_next_token.json b/pleroma_bot/tests/test_files/sample_data/tweets_v2_next_token.json index 0d182cc..6fb21d8 100644 --- a/pleroma_bot/tests/test_files/sample_data/tweets_v2_next_token.json +++ b/pleroma_bot/tests/test_files/sample_data/tweets_v2_next_token.json @@ -144,7 +144,7 @@ "id": "1347782491248054272" } ], - "author_id": "81290806", + "author_id": "1320506197913542656", "lang": "en", "possibly_sensitive": false, "public_metrics": { diff --git a/pleroma_bot/tests/test_files/sample_data/tweets_v2_next_token2.json b/pleroma_bot/tests/test_files/sample_data/tweets_v2_next_token2.json index 6cfd01c..1f0389b 100644 --- a/pleroma_bot/tests/test_files/sample_data/tweets_v2_next_token2.json +++ b/pleroma_bot/tests/test_files/sample_data/tweets_v2_next_token2.json @@ -7,7 +7,7 @@ "1323049466027479040" ] }, - "text": "Poll", + "text": "Github.com Poll", "public_metrics": { "retweet_count": 0, "reply_count": 0, @@ -144,7 +144,7 @@ "id": "1347782491248054272" } ], - "author_id": "81290806", + "author_id": "1320506197913542656", "lang": "en", "possibly_sensitive": false, "public_metrics": { diff --git a/pleroma_bot/tests/test_files/sample_data/twitter_info.json b/pleroma_bot/tests/test_files/sample_data/twitter_info.json index 56d5749..1796997 100644 --- a/pleroma_bot/tests/test_files/sample_data/twitter_info.json +++ b/pleroma_bot/tests/test_files/sample_data/twitter_info.json @@ -72,7 +72,7 @@ "profile_background_tile":false, "profile_banner_url": "https:\/\/pbs.twimg.com\/profile_banners\/1320506197913542656\/1604883750", "profile_image_url":"http:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png", - "profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile_normal.png", + "profile_image_url_https":"https:\/\/abs.twimg.com\/sticky\/default_profile_images\/default_profile.png", "profile_link_color":"1DA1F2", "profile_sidebar_border_color":"C0DEED", "profile_sidebar_fill_color":"DDEEF6", diff --git a/pleroma_bot/tests/test_files/sample_data/user_v2.json b/pleroma_bot/tests/test_files/sample_data/user_v2.json new file mode 100644 index 0000000..3d8d697 --- /dev/null +++ b/pleroma_bot/tests/test_files/sample_data/user_v2.json @@ -0,0 +1 @@ +{"includes": {"tweets": [{"text": "Poll", "id": "1323049466837032961"}]},"data":{"profile_image_url":"https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png","url":"","id":"1320506197913542656","description":"My bio text.\nMultiple lines,\neven.","username":"BotPleroma","name":"test-pleroma-bot","protected":true,"pinned_tweet_id":"1323049466837032961","verified":false,"created_at":"2020-10-25T23:23:13.000Z"},"errors":[{"value":"1323049466837032961","detail":"Could not find tweet with pinned_tweet_id: [1323049466837032961].","title":"Not Found Error","resource_type":"tweet","parameter":"pinned_tweet_id","resource_id":"1323049466837032961","type":"https://api.twitter.com/2/problems/resource-not-found"}]} \ No newline at end of file diff --git a/pleroma_bot/tests/test_files/sample_data/user_v2_HackerNews.json b/pleroma_bot/tests/test_files/sample_data/user_v2_HackerNews.json new file mode 100644 index 0000000..5a06f81 --- /dev/null +++ b/pleroma_bot/tests/test_files/sample_data/user_v2_HackerNews.json @@ -0,0 +1 @@ +{"includes": {"tweets": [{"text": "Poll", "id": "1323049466837032961"}]},"data":{"profile_image_url":"https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png","url":"","id":"1320506197913542656","description":"My bio text.\nMultiple lines,\neven.","username":"HackerNews","name":"test-pleroma-bot","protected":true,"pinned_tweet_id":"1323049466837032961","verified":false,"created_at":"2020-10-25T23:23:13.000Z"},"errors":[{"value":"1323049466837032961","detail":"Could not find tweet with pinned_tweet_id: [1323049466837032961].","title":"Not Found Error","resource_type":"tweet","parameter":"pinned_tweet_id","resource_id":"1323049466837032961","type":"https://api.twitter.com/2/problems/resource-not-found"}]} \ No newline at end of file diff --git a/pleroma_bot/tests/test_files/sample_data/user_v2_KyleBosman.json b/pleroma_bot/tests/test_files/sample_data/user_v2_KyleBosman.json new file mode 100644 index 0000000..85e67b0 --- /dev/null +++ b/pleroma_bot/tests/test_files/sample_data/user_v2_KyleBosman.json @@ -0,0 +1 @@ +{"includes": {"tweets": [{"text": "Poll", "id": "1323049466837032961"}]},"data":{"profile_image_url":"https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png","url":"","id":"1320506197913542656","description":"My bio text.\nMultiple lines,\neven.","username":"KyleBosman","name":"test-pleroma-bot","protected":true,"pinned_tweet_id":"1323049466837032961","verified":false,"created_at":"2020-10-25T23:23:13.000Z"},"errors":[{"value":"1323049466837032961","detail":"Could not find tweet with pinned_tweet_id: [1323049466837032961].","title":"Not Found Error","resource_type":"tweet","parameter":"pinned_tweet_id","resource_id":"1323049466837032961","type":"https://api.twitter.com/2/problems/resource-not-found"}]} \ No newline at end of file diff --git a/pleroma_bot/tests/test_files/sample_data/user_v2_arstechnica.json b/pleroma_bot/tests/test_files/sample_data/user_v2_arstechnica.json new file mode 100644 index 0000000..4d2d794 --- /dev/null +++ b/pleroma_bot/tests/test_files/sample_data/user_v2_arstechnica.json @@ -0,0 +1 @@ +{"includes": {"tweets": [{"text": "Poll", "id": "1323049466837032961"}]},"data":{"profile_image_url":"https://abs.twimg.com/sticky/default_profile_images/default_profile_normal.png","url":"","id":"1320506197913542656","description":"My bio text.\nMultiple lines,\neven.","username":"arstechnica","name":"test-pleroma-bot","protected":true,"pinned_tweet_id":"1323049466837032961","verified":false,"created_at":"2020-10-25T23:23:13.000Z"},"errors":[{"value":"1323049466837032961","detail":"Could not find tweet with pinned_tweet_id: [1323049466837032961].","title":"Not Found Error","resource_type":"tweet","parameter":"pinned_tweet_id","resource_id":"1323049466837032961","type":"https://api.twitter.com/2/problems/resource-not-found"}]} \ No newline at end of file diff --git a/pleroma_bot/tests/test_logger.py b/pleroma_bot/tests/test_logger.py index 6d9e960..a05df39 100644 --- a/pleroma_bot/tests/test_logger.py +++ b/pleroma_bot/tests/test_logger.py @@ -31,12 +31,15 @@ def test_unpin_pleroma_logger(sample_users, mock_request, caplog): ) empty_file = os.path.join(os.getcwd(), 'empty.txt') open(empty_file, 'a').close() - pinned_file = os.path.join(sample_user_obj.user_path, - "pinned_id_pleroma.txt") - shutil.copy(empty_file, pinned_file) - sample_user_obj.unpin_pleroma(pinned_file) - os.remove(pinned_file) - os.remove(empty_file) + for t_user in sample_user_obj.twitter_username: + pinned_file = os.path.join( + sample_user_obj.user_path[t_user], + "pinned_id_pleroma.txt" + ) + shutil.copy(empty_file, pinned_file) + sample_user_obj.unpin_pleroma(pinned_file) + os.remove(pinned_file) + os.remove(empty_file) assert 'Pinned post not found. Giving up unpinning...' in caplog.text @@ -57,21 +60,27 @@ def test_main_exception_logger(global_mock, sample_users, caplog): shutil.copy(backup_config, prev_config) for sample_user in sample_users: sample_user_obj = sample_user['user_obj'] - pinned_path = os.path.join(os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id.txt') - pinned_pleroma = os.path.join(os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id_pleroma.txt') - if os.path.isfile(pinned_path): - os.remove(pinned_path) - if os.path.isfile(pinned_pleroma): - os.remove(pinned_pleroma) - # Restore config - if os.path.isfile(backup_config): - shutil.copy(backup_config, prev_config) + for t_user in sample_user_obj.twitter_username: + idx = sample_user_obj.twitter_username.index(t_user) + pinned_path = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id.txt' + ) + pinned_pleroma = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id_pleroma.txt' + ) + if os.path.isfile(pinned_path): + os.remove(pinned_path) + if os.path.isfile(pinned_pleroma): + os.remove(pinned_pleroma) + # Restore config + if os.path.isfile(backup_config): + shutil.copy(backup_config, prev_config) mock.reset_mock() assert 'Exception occurred\nTraceback' in caplog.text @@ -140,33 +149,36 @@ def test_post_pleroma_media_size_logger( sample_user_obj = User( user_item, users_file_size['config'], os.getcwd() ) - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] - - with caplog.at_level(logging.ERROR): - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - if hasattr(sample_user_obj, "file_max_size"): - error_msg = ( - f'Attachment exceeded config file size ' - f'limit ({sample_user_obj.file_max_size})' - ) - assert error_msg in caplog.text - assert 'File size: 1.45MB' in caplog.text - assert 'Ignoring attachment and continuing...' in caplog.text + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets("v2", t_user=t_user) + assert tweets_v2 == mock_request['sample_data']['tweets_v2'] + tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] - for tweet in tweets_to_post['data']: - # Clean up - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] - ) - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - os.remove(file_path) + with caplog.at_level(logging.ERROR): + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + if hasattr(sample_user_obj, "file_max_size"): + error_msg = ( + f'Attachment exceeded config file size ' + f'limit ({sample_user_obj.file_max_size})' + ) + assert error_msg in caplog.text + assert 'File size: 1.45MB' in caplog.text + ignore = 'Ignoring attachment and continuing...' + assert ignore in caplog.text + + for tweet in tweets_to_post['data']: + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) return mock @@ -175,29 +187,30 @@ def test_get_instance_info_mastodon(global_mock, sample_users, caplog): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - sample_user_obj.display_name = random_string(50) - mock.get(f"{test_user.pleroma_base_url}/api/v1/instance", - json={'version': '3.2.1'}, - status_code=200) - rich_text_orig = False - assert len(sample_user_obj.display_name) == 50 - if hasattr(sample_user_obj, "rich_text"): - if sample_user_obj.rich_text: - rich_text_orig = True - with caplog.at_level(logging.DEBUG): - sample_user_obj._get_instance_info() - assert 'Assuming target instance is Mastodon...' in caplog.text - if rich_text_orig: - log_msg_rich_text = ( - "Mastodon doesn't support rich text. Disabling it..." - ) - log_msg_display_name = ( - "Mastodon doesn't support display names longer than 30 " - "characters, truncating it and trying again..." - ) - assert log_msg_rich_text in caplog.text - assert log_msg_display_name in caplog.text - assert len(sample_user_obj.display_name) == 30 + for t_user in sample_user_obj.twitter_username: + sample_user_obj.display_name = {t_user: random_string(50)} + mock.get(f"{test_user.pleroma_base_url}/api/v1/instance", + json={'version': '3.2.1'}, + status_code=200) + rich_text_orig = False + assert len(sample_user_obj.display_name[t_user]) == 50 + if hasattr(sample_user_obj, "rich_text"): + if sample_user_obj.rich_text: + rich_text_orig = True + with caplog.at_level(logging.DEBUG): + sample_user_obj._get_instance_info() + assert 'Assuming target instance is Mastodon...' in caplog.text + if rich_text_orig: + log_msg_rich_text = ( + "Mastodon doesn't support rich text. Disabling it..." + ) + log_msg_display_name = ( + "Mastodon doesn't support display names longer " + "than 30 characters, truncating it and trying again..." + ) + assert log_msg_rich_text in caplog.text + assert log_msg_display_name in caplog.text + assert len(sample_user_obj.display_name[t_user]) == 30 def test_force_date_logger(sample_users, monkeypatch, caplog): diff --git a/pleroma_bot/tests/test_run.py b/pleroma_bot/tests/test_run.py index 7493482..edb87fe 100644 --- a/pleroma_bot/tests/test_run.py +++ b/pleroma_bot/tests/test_run.py @@ -1,9 +1,11 @@ import os +import re import sys import shutil import hashlib import logging import urllib.parse +import multiprocessing as mp from unittest.mock import patch from datetime import datetime, timedelta from urllib import parse @@ -11,8 +13,8 @@ from conftest import get_config_users from pleroma_bot import cli, User -from pleroma_bot._utils import random_string -from pleroma_bot._utils import guess_type +from pleroma_bot._utils import random_string, previous_and_next, guess_type +from pleroma_bot._utils import process_parallel, process_archive def test_random_string(): @@ -31,8 +33,9 @@ def test_user_replace_vars_in_str(sample_users): test_user = UserTemplate() for sample_user in sample_users: user_obj = sample_user['user_obj'] - replace = user_obj.replace_vars_in_str(test_user.replace_str) - assert replace == sample_user['user_obj'].twitter_url + for t_user in user_obj.twitter_username: + replace = user_obj.replace_vars_in_str(test_user.replace_str) + assert replace == sample_user['user_obj'].twitter_url[t_user] def test_user_replace_vars_in_str_var(sample_users): @@ -43,11 +46,12 @@ def test_user_replace_vars_in_str_var(sample_users): test_user = UserTemplate() for sample_user in sample_users: user_obj = sample_user['user_obj'] - replace = user_obj.replace_vars_in_str( - test_user.replace_str, - "twitter_url" - ) - assert replace == sample_user['user_obj'].twitter_url + for t_user in user_obj.twitter_username: + replace = user_obj.replace_vars_in_str( + test_user.replace_str, + "twitter_url" + ) + assert replace == sample_user['user_obj'].twitter_url[t_user] def test_replace_vars_in_str_local(monkeypatch, sample_users): @@ -121,151 +125,171 @@ def test_check_pinned_tweet(sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - pinned = sample_user_obj.pinned_tweet_id - assert pinned == test_user.pinned - mock.get(f"{test_user.twitter_base_url_v2}/tweets/{pinned}" - f"?poll.fields=duration_minutes%2Cend_datetime%2Cid%2C" - f"options%2Cvoting_status&media.fields=duration_ms%2C" - f"height%2Cmedia_key%2Cpreview_image_url%2Ctype%2Curl%2C" - f"width%2Cpublic_metrics&expansions=attachments.poll_ids" - f"%2Cattachments.media_keys%2Cauthor_id%2C" - f"entities.mentions.username%2Cgeo.place_id%2C" - f"in_reply_to_user_id%2Creferenced_tweets.id%2C" - f"referenced_tweets.id.author_id&tweet.fields=attachments" - f"%2Cauthor_id%2Ccontext_annotations%2Cconversation_id%2" - f"Ccreated_at%2Centities%2Cgeo%2Cid%2Cin_reply_to_user_id" - f"%2Clang%2Cpublic_metrics%2Cpossibly_sensitive%2C" - f"referenced_tweets%2Csource%2Ctext%2Cwithheld", - json=mock_request['sample_data']['pinned_tweet'], - status_code=200) - mock.get(f"{test_user.twitter_base_url_v2}/tweets?ids={pinned}" - f"&expansions=attachments.poll_ids" - f"&poll.fields=duration_minutes%2Coptions", - json=mock_request['sample_data']['poll'], - status_code=200) - pinned_file = os.path.join( - os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id.txt' - ) - with open(pinned_file, "w") as f: - f.write(test_user.pinned + "\n") - sample_user_obj.check_pinned() - pinned_path = os.path.join(os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id.txt') - pinned_pleroma = os.path.join(os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id_pleroma.txt') - with open(pinned_path, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == test_user.pinned - - # Pinned -> Pinned (different ID) - pinned_url = ( - f"{test_user.twitter_base_url_v2}/users/by/username/" - f"{sample_user_obj.twitter_username}" - ) - mock.get(pinned_url, - json=mock_request['sample_data']['pinned_2'], - status_code=200) - new_pin_id = sample_user_obj._get_pinned_tweet_id() - sample_user_obj.pinned_tweet_id = new_pin_id - pinned = sample_user_obj.pinned_tweet_id - mock.get(f"{test_user.twitter_base_url_v2}/tweets/{pinned}" - f"?poll.fields=duration_minutes%2Cend_datetime%2Cid%2C" - f"options%2Cvoting_status&media.fields=duration_ms%2C" - f"height%2Cmedia_key%2Cpreview_image_url%2Ctype%2Curl%2C" - f"width%2Cpublic_metrics&expansions=attachments.poll_ids" - f"%2Cattachments.media_keys%2Cauthor_id%2C" - f"entities.mentions.username%2Cgeo.place_id%2C" - f"in_reply_to_user_id%2Creferenced_tweets.id%2C" - f"referenced_tweets.id.author_id&tweet.fields=attachments" - f"%2Cauthor_id%2Ccontext_annotations%2Cconversation_id%2" - f"Ccreated_at%2Centities%2Cgeo%2Cid%2Cin_reply_to_user_id" - f"%2Clang%2Cpublic_metrics%2Cpossibly_sensitive%2C" - f"referenced_tweets%2Csource%2Ctext%2Cwithheld", - json=mock_request['sample_data']['pinned_tweet_2'], - status_code=200) - mock.get(f"{test_user.twitter_base_url_v2}/tweets?ids={pinned}" - f"&expansions=attachments.poll_ids" - f"&poll.fields=duration_minutes%2Coptions", - json=mock_request['sample_data']['poll_2'], - status_code=200) - sample_user_obj.check_pinned() - with open(pinned_path, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == test_user.pinned_2 - id_pleroma = test_user.pleroma_pinned - with open(pinned_pleroma, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == id_pleroma - - # Pinned -> None - mock.get(f"{test_user.twitter_base_url_v2}/users/by/username/" - f"{sample_user_obj.twitter_username}", - json=mock_request['sample_data']['no_pinned'], - status_code=200) - new_pin_id = sample_user_obj._get_pinned_tweet_id() - sample_user_obj.pinned_tweet_id = new_pin_id - sample_user_obj.check_pinned() - with open(pinned_path, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == '' - with open(pinned_pleroma, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == '' - history = mock.request_history - unpin_url = ( - f"{sample_user_obj.pleroma_base_url}" - f"/api/v1/statuses/{test_user.pleroma_pinned}/unpin" - ) - assert unpin_url == history[-1].url - - # None -> None - sample_user_obj.check_pinned() - with open(pinned_path, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == '' - with open(pinned_pleroma, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == '' - - # None -> Pinned - pinned_url = ( - f"{test_user.twitter_base_url_v2}/users/by/username/" - f"{sample_user_obj.twitter_username}" - ) - mock.get(pinned_url, - json=mock_request['sample_data']['pinned'], - status_code=200) - new_pin_id = sample_user_obj._get_pinned_tweet_id() - sample_user_obj.pinned_tweet_id = new_pin_id - pinned = sample_user_obj.pinned_tweet_id - mock.get(f"{test_user.twitter_base_url_v2}/tweets/{pinned}" - f"?poll.fields=duration_minutes%2Cend_datetime%2Cid%2C" - f"options%2Cvoting_status&media.fields=duration_ms%2C" - f"height%2Cmedia_key%2Cpreview_image_url%2Ctype%2Curl%2C" - f"width%2Cpublic_metrics&expansions=attachments.poll_ids" - f"%2Cattachments.media_keys%2Cauthor_id%2C" - f"entities.mentions.username%2Cgeo.place_id%2C" - f"in_reply_to_user_id%2Creferenced_tweets.id%2C" - f"referenced_tweets.id.author_id&tweet.fields=attachments" - f"%2Cauthor_id%2Ccontext_annotations%2Cconversation_id%2" - f"Ccreated_at%2Centities%2Cgeo%2Cid%2Cin_reply_to_user_id" - f"%2Clang%2Cpublic_metrics%2Cpossibly_sensitive%2C" - f"referenced_tweets%2Csource%2Ctext%2Cwithheld", - json=mock_request['sample_data']['pinned_tweet'], - status_code=200) - mock.get(f"{test_user.twitter_base_url_v2}/tweets?ids={pinned}" - f"&expansions=attachments.poll_ids" - f"&poll.fields=duration_minutes%2Coptions", - json=mock_request['sample_data']['poll'], - status_code=200) - sample_user_obj.check_pinned() - with open(pinned_path, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == test_user.pinned - id_pleroma = test_user.pleroma_pinned - with open(pinned_pleroma, 'r', encoding='utf8') as f: - assert f.readline().rstrip() == id_pleroma - os.remove(pinned_path) - os.remove(pinned_pleroma) + for t_user in sample_user_obj.twitter_username: + idx = sample_user_obj.twitter_username.index(t_user) + pinned = sample_user_obj.pinned_tweet_id + assert pinned == test_user.pinned + mock.get(f"{test_user.twitter_base_url_v2}/tweets/{pinned}" + f"?poll.fields=duration_minutes%2Cend_datetime%2Cid" + f"%2Coptions%2Cvoting_status&media.fields=duration_" + f"ms%2Cheight%2Cmedia_key%2Cpreview_image_url%2C" + f"type%2Curl%2Cwidth%2Cpublic_metrics&expansions=" + f"attachments.poll_ids%2Cattachments.media_keys%2C" + f"author_id%2Centities.mentions.username%2C" + f"geo.place_id%2Cin_reply_to_user_id%2C" + f"referenced_tweets.id%2Creferenced_tweets.id." + f"author_id&tweet.fields=attachments%2Cauthor_id" + f"%2Ccontext_annotations%2Cconversation_id%2C" + f"created_at%2Centities%2Cgeo%2Cid%2C" + f"in_reply_to_user_id%2Clang%2Cpublic_metrics%" + f"2Cpossibly_sensitive%2Creferenced_tweets%2C" + f"source%2Ctext%2Cwithheld", + json=mock_request['sample_data']['pinned_tweet'], + status_code=200) + mock.get(f"{test_user.twitter_base_url_v2}/tweets?ids={pinned}" + f"&expansions=attachments.poll_ids" + f"&poll.fields=duration_minutes%2Coptions", + json=mock_request['sample_data']['poll'], + status_code=200) + pinned_file = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id.txt' + ) + with open(pinned_file, "w") as f: + f.write(test_user.pinned + "\n") + sample_user_obj.check_pinned() + pinned_path = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id.txt' + ) + pinned_pleroma = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id_pleroma.txt' + ) + with open(pinned_path, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == test_user.pinned + + # Pinned -> Pinned (different ID) + pinned_url = ( + f"{test_user.twitter_base_url_v2}/users/by/username/" + f"{sample_user_obj.twitter_username[idx]}" + ) + mock.get(pinned_url, + json=mock_request['sample_data']['pinned_2'], + status_code=200) + new_pin_id = sample_user_obj._get_pinned_tweet_id() + sample_user_obj.pinned_tweet_id = new_pin_id + pinned = sample_user_obj.pinned_tweet_id + mock.get(f"{test_user.twitter_base_url_v2}/tweets/{pinned}" + f"?poll.fields=duration_minutes%2C" + f"end_datetime%2Cid%2Coptions%2C" + f"voting_status&media.fields=duration_ms%2C" + f"height%2Cmedia_key%2C" + f"preview_image_url%2Ctype%2Curl%2C" + f"width%2Cpublic_metrics&" + f"expansions=attachments.poll_ids" + f"%2Cattachments.media_keys%2Cauthor_id%2C" + f"entities.mentions.username%2Cgeo.place_id%2C" + f"in_reply_to_user_id%2Creferenced_tweets.id%2C" + f"referenced_tweets.id.author_" + f"id&tweet.fields=attachments" + f"%2Cauthor_id%2Ccontext_" + f"annotations%2Cconversation_id%2" + f"Ccreated_at%2Centities%2Cgeo%2C" + f"id%2Cin_reply_to_user_id" + f"%2Clang%2Cpublic_metrics%2Cpossibly_sensitive%2C" + f"referenced_tweets%2Csource%2Ctext%2Cwithheld", + json=mock_request['sample_data']['pinned_tweet_2'], + status_code=200) + mock.get(f"{test_user.twitter_base_url_v2}/tweets?ids={pinned}" + f"&expansions=attachments.poll_ids" + f"&poll.fields=duration_minutes%2Coptions", + json=mock_request['sample_data']['poll_2'], + status_code=200) + sample_user_obj.check_pinned() + with open(pinned_path, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == test_user.pinned_2 + id_pleroma = test_user.pleroma_pinned + with open(pinned_pleroma, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == id_pleroma + + # Pinned -> None + mock.get(f"{test_user.twitter_base_url_v2}/users/by/username/" + f"{sample_user_obj.twitter_username[idx]}", + json=mock_request['sample_data']['no_pinned'], + status_code=200) + new_pin_id = sample_user_obj._get_pinned_tweet_id() + sample_user_obj.pinned_tweet_id = new_pin_id + sample_user_obj.check_pinned() + with open(pinned_path, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == '' + with open(pinned_pleroma, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == '' + history = mock.request_history + unpin_url = ( + f"{sample_user_obj.pleroma_base_url}" + f"/api/v1/statuses/{test_user.pleroma_pinned}/unpin" + ) + assert unpin_url == history[-1].url + + # None -> None + sample_user_obj.check_pinned() + with open(pinned_path, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == '' + with open(pinned_pleroma, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == '' + + # None -> Pinned + pinned_url = ( + f"{test_user.twitter_base_url_v2}/users/by/username/" + f"{sample_user_obj.twitter_username[idx]}" + ) + mock.get(pinned_url, + json=mock_request['sample_data']['pinned'], + status_code=200) + new_pin_id = sample_user_obj._get_pinned_tweet_id() + sample_user_obj.pinned_tweet_id = new_pin_id + pinned = sample_user_obj.pinned_tweet_id + mock.get(f"{test_user.twitter_base_url_v2}/tweets/{pinned}" + f"?poll.fields=duration_minutes%2Cend_" + f"datetime%2Cid%2C" + f"options%2Cvoting_status&media.fields=duration_ms%2C" + f"height%2Cmedia_key%2Cpreview" + f"_image_url%2Ctype%2Curl%2C" + f"width%2Cpublic_metrics&expans" + f"ions=attachments.poll_ids" + f"%2Cattachments.media_keys%2Cauthor_id%2C" + f"entities.mentions.username%2Cgeo.place_id%2C" + f"in_reply_to_user_id%2Creferenced_tweets.id%2C" + f"referenced_tweets.id.author_id" + f"&tweet.fields=attachments" + f"%2Cauthor_id%2Ccontext_annotat" + f"ions%2Cconversation_id%2" + f"Ccreated_at%2Centities%2Cgeo%2" + f"Cid%2Cin_reply_to_user_id" + f"%2Clang%2Cpublic_metrics%2Cpossibly_sensitive%2C" + f"referenced_tweets%2Csource%2Ctext%2Cwithheld", + json=mock_request['sample_data']['pinned_tweet'], + status_code=200) + mock.get(f"{test_user.twitter_base_url_v2}/tweets?ids={pinned}" + f"&expansions=attachments.poll_ids" + f"&poll.fields=duration_minutes%2Coptions", + json=mock_request['sample_data']['poll'], + status_code=200) + sample_user_obj.check_pinned() + with open(pinned_path, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == test_user.pinned + id_pleroma = test_user.pleroma_pinned + with open(pinned_pleroma, 'r', encoding='utf8') as f: + assert f.readline().rstrip() == id_pleroma + os.remove(pinned_path) + os.remove(pinned_pleroma) def test_get_date_last_pleroma_post(sample_users): @@ -322,6 +346,15 @@ def test_guess_type(rootdir): assert 'image/gif' == guess_type(gif) +def test_process_archive(rootdir): + test_files_dir = os.path.join(rootdir, 'test_files') + sample_data_dir = os.path.join(test_files_dir, 'sample_data') + media_dir = os.path.join(sample_data_dir, 'media') + archive = os.path.join(media_dir, 'twitter-archive.zip') + tweets = process_archive(archive) + assert tweets is not None + + def test_get_twitter_info(mock_request, sample_users): """ Check that _get_twitter_info retrieves the correct profile image and banner @@ -330,16 +363,17 @@ def test_get_twitter_info(mock_request, sample_users): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - twitter_info = mock_request['sample_data']['twitter_info'] - banner_url = f"{twitter_info['profile_banner_url']}/1500x500" - profile_pic_url = twitter_info['profile_image_url_https'] + for t_user in sample_user_obj.twitter_username: + twitter_info = mock_request['sample_data']['twitter_info'] + banner_url = f"{twitter_info['profile_banner_url']}/1500x500" + profile_pic_url = twitter_info['profile_image_url_https'] - sample_user_obj._get_twitter_info() + sample_user_obj._get_twitter_info() - p_banner_url = sample_user_obj.profile_banner_url - p_image_url = sample_user_obj.profile_image_url - assert banner_url == p_banner_url - assert profile_pic_url == p_image_url + p_banner_url = sample_user_obj.profile_banner_url + p_image_url = sample_user_obj.profile_image_url + assert banner_url == p_banner_url[t_user] + assert profile_pic_url == p_image_url[t_user] return mock @@ -350,31 +384,41 @@ def test_update_pleroma(sample_users, rootdir): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - test_files_dir = os.path.join(rootdir, 'test_files') - sample_data_dir = os.path.join(test_files_dir, 'sample_data') - media_dir = os.path.join(sample_data_dir, 'media') - - banner = os.path.join(media_dir, 'banner.jpg') - profile_banner = open(banner, 'rb') - profile_banner_content = profile_banner.read() - profile_banner.close() - - profile_pic = os.path.join(media_dir, 'default_profile_normal.png') - profile_image = open(profile_pic, 'rb') - profile_image_content = profile_image.read() - profile_image.close() - - sample_user_obj.update_pleroma() - - t_profile_banner = open(sample_user_obj.header_path, 'rb') - t_profile_banner_content = t_profile_banner.read() - t_profile_banner.close() - - t_profile_image = open(sample_user_obj.avatar_path, 'rb') - t_profile_image_content = t_profile_image.read() - t_profile_image.close() - assert t_profile_banner_content == profile_banner_content - assert t_profile_image_content == profile_image_content + for t_user in sample_user_obj.twitter_username: + test_files_dir = os.path.join(rootdir, 'test_files') + sample_data_dir = os.path.join(test_files_dir, 'sample_data') + media_dir = os.path.join(sample_data_dir, 'media') + + banner = os.path.join(media_dir, 'banner.jpg') + profile_banner = open(banner, 'rb') + profile_banner_content = profile_banner.read() + profile_banner.close() + + profile_pic = os.path.join( + media_dir, + 'default_profile_normal.png' + ) + profile_image = open(profile_pic, 'rb') + profile_image_content = profile_image.read() + profile_image.close() + + sample_user_obj.update_pleroma() + + t_profile_banner = open( + sample_user_obj.header_path[t_user], + 'rb' + ) + t_profile_banner_content = t_profile_banner.read() + t_profile_banner.close() + + t_profile_image = open( + sample_user_obj.avatar_path[t_user], + 'rb' + ) + t_profile_image_content = t_profile_image.read() + t_profile_image.close() + assert t_profile_banner_content == profile_banner_content + assert t_profile_image_content == profile_image_content return mock @@ -394,11 +438,11 @@ def test_post_pleroma_media(rootdir, sample_users, mock_request): tweet_folder = os.path.join( sample_user_obj.tweets_temp_path, test_user.pinned ) + os.makedirs(tweet_folder, exist_ok=True) shutil.copy(png, tweet_folder) shutil.copy(svg, tweet_folder) shutil.copy(mp4, tweet_folder) shutil.copy(gif, tweet_folder) - attach_number = len(os.listdir(tweet_folder)) sample_user_obj.post_pleroma( (test_user.pinned, "", ""), None, False ) @@ -424,6 +468,7 @@ def test_post_pleroma_media(rootdir, sample_users, mock_request): id_media = mock_media['id'] assert id_media in history[-1].text dict_history = urllib.parse.parse_qs(history[-1].text) + attach_number = len(os.listdir(tweet_folder)) assert len(dict_history['media_ids[]']) == attach_number for media in dict_history['media_ids[]']: assert media == id_media @@ -437,12 +482,17 @@ def test_get_tweets(sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] + for t_user in sample_user_obj.twitter_username: + start_time = sample_user_obj.get_date_last_pleroma_post() + _ = sample_user_obj._get_tweets( + "v2", start_time=start_time, t_user=t_user + ) + tweets_v2 = sample_user_obj._get_tweets("v2", t_user=t_user) + assert tweets_v2 == mock_request['sample_data']['tweets_v2'] + tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] return mock @@ -455,17 +505,20 @@ def test_get_tweets_next_token(sample_users, mock_request): json=mock_request['sample_data']['tweets_v2_next_token'], status_code=200) sample_user_obj = sample_user['user_obj'] - tweets_v2 = sample_user_obj._get_tweets("v2") - assert 10 == len(tweets_v2["data"]) - - mock.get(f"{test_user.twitter_base_url_v2}/users/2244994945" - f"/tweets", - json=mock_request['sample_data']['tweets_v2_next_token2'], - status_code=200) - - sample_user_obj = sample_user['user_obj'] - tweets_v2 = sample_user_obj._get_tweets("v2") - assert 10 == len(tweets_v2["data"]) + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets("v2", t_user=t_user) + assert 10 == len(tweets_v2["data"]) + + mock.get( + f"{test_user.twitter_base_url_v2}" + f"/users/2244994945" + f"/tweets", + json=mock_request['sample_data']['tweets_v2_next_token2'], + status_code=200) + + sample_user_obj = sample_user['user_obj'] + tweets_v2 = sample_user_obj._get_tweets("v2", t_user=t_user) + assert 10 == len(tweets_v2["data"]) return mock @@ -474,62 +527,66 @@ def test_process_tweets(rootdir, sample_users, mock_request): for sample_user in sample_users: with sample_user['mock'] as mock: sample_user_obj = sample_user['user_obj'] - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] - test_files_dir = os.path.join(rootdir, 'test_files') - sample_data_dir = os.path.join(test_files_dir, 'sample_data') - media_dir = os.path.join(sample_data_dir, 'media') - mp4 = os.path.join(media_dir, 'video.mp4') - gif = os.path.join(media_dir, "animated_gif.gif") - png = os.path.join(media_dir, 'image.png') - - gif_file = open(gif, 'rb') - gif_content = gif_file.read() - gif_hash = hashlib.sha256(gif_content).hexdigest() - gif_file.close() - - png_file = open(png, 'rb') - png_content = png_file.read() - png_hash = hashlib.sha256(png_content).hexdigest() - png_file.close() - - mp4_file = open(mp4, 'rb') - mp4_content = mp4_file.read() - mp4_hash = hashlib.sha256(mp4_content).hexdigest() - mp4_file.close() - - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - - for tweet in tweets_to_post['data']: - # Test poll retrieval - if tweet['id'] == test_user.pinned: - poll = mock_request['sample_data']['poll'] - options = poll['includes']['polls'][0]['options'] - duration = poll['includes']['polls'][0]['duration_minutes'] - assert len(tweet['polls']['options']) == len(options) - assert tweet['polls']['expires_in'] == duration * 60 - # Test download - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] - ) - dict_hash = { - '0.mp4': mp4_hash, - '0.png': png_hash, - '0.gif': gif_hash - } - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - f = open(file_path, 'rb') - file_content = f.read() - file_hash = hashlib.sha256(file_content).hexdigest() - f.close() - assert file_hash == dict_hash[file] - os.remove(file_path) + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets("v2", t_user=t_user) + assert tweets_v2 == mock_request['sample_data']['tweets_v2'] + tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] + test_files_dir = os.path.join(rootdir, 'test_files') + sample_data_dir = os.path.join(test_files_dir, 'sample_data') + media_dir = os.path.join(sample_data_dir, 'media') + mp4 = os.path.join(media_dir, 'video.mp4') + gif = os.path.join(media_dir, "animated_gif.gif") + png = os.path.join(media_dir, 'image.png') + + gif_file = open(gif, 'rb') + gif_content = gif_file.read() + gif_hash = hashlib.sha256(gif_content).hexdigest() + gif_file.close() + + png_file = open(png, 'rb') + png_content = png_file.read() + png_hash = hashlib.sha256(png_content).hexdigest() + png_file.close() + + mp4_file = open(mp4, 'rb') + mp4_content = mp4_file.read() + mp4_hash = hashlib.sha256(mp4_content).hexdigest() + mp4_file.close() + + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + + for tweet in tweets_to_post['data']: + # Test poll retrieval + if tweet['id'] == test_user.pinned: + poll = mock_request['sample_data']['poll'] + options = poll['includes']['polls'][0]['options'] + polls = poll['includes']['polls'] + duration = polls[0]['duration_minutes'] + assert len(tweet['polls']['options']) == len(options) + assert tweet['polls']['expires_in'] == duration * 60 + # Test download + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + dict_hash = { + '0.mp4': mp4_hash, + '0.png': png_hash, + '0.gif': gif_hash + } + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + f = open(file_path, 'rb') + file_cont = f.read() + sha256 = hashlib.sha256(file_cont) + file_hash = sha256.hexdigest() + f.close() + assert file_hash == dict_hash[file] + os.remove(file_path) return mock @@ -542,31 +599,38 @@ def test_include_rts(sample_users, mock_request): sample_user_obj = User( user_item, config_users['config'], os.getcwd() ) - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets( + "v2", t_user=t_user + ) + sample_data = mock_request['sample_data'] + assert tweets_v2 == sample_data['tweets_v2'] + tweet = sample_user_obj._get_tweets( + "v1.1", test_user.pinned + ) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - retweet_found = False - for tweet in tweets_to_post['data']: - if not sample_user_obj.include_rts: - assert not tweet["text"].startswith("RT") - if tweet["text"].startswith("RT"): - retweet_found = True - # Clean up - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] - ) - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - os.remove(file_path) - if retweet_found: - assert sample_user_obj.include_rts + retweet_found = False + for tweet in tweets_to_post['data']: + if not sample_user_obj.include_rts: + assert not tweet["text"].startswith("RT") + if tweet["text"].startswith("RT"): + retweet_found = True + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) + if retweet_found: + assert sample_user_obj.include_rts return mock @@ -579,48 +643,55 @@ def test_include_replies(sample_users, mock_request): sample_user_obj = User( user_item, config_users['config'], os.getcwd() ) - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets( + "v2", t_user=t_user + ) + sample_data = mock_request['sample_data'] + assert tweets_v2 == sample_data['tweets_v2'] + tweet = sample_user_obj._get_tweets( + "v1.1", test_user.pinned + ) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - reply_found = False + reply_found = False - for tweet in tweets_to_post['data']: - if not sample_user_obj.include_replies: + for tweet in tweets_to_post['data']: + if not sample_user_obj.include_replies: + if ( + hasattr(sample_user_obj, "rich_text") + and sample_user_obj.rich_text + ): + assert not tweet["text"].startswith("[@") + else: + assert not tweet["text"].startswith("@") + try: + for reference in tweet["referenced_tweets"]: + if reference["type"] == "replied_to": + reply_found = True + break + except KeyError: + pass if ( - hasattr(sample_user_obj, "rich_text") - and sample_user_obj.rich_text + tweet["text"].startswith("@") + or tweet["text"].startswith("[@") ): - assert not tweet["text"].startswith("[@") - else: - assert not tweet["text"].startswith("@") - try: - for reference in tweet["referenced_tweets"]: - if reference["type"] == "replied_to": - reply_found = True - break - except KeyError: - pass - if ( - tweet["text"].startswith("@") - or tweet["text"].startswith("[@") - ): - reply_found = True - # Clean up - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] - ) - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - os.remove(file_path) - if reply_found: - assert sample_user_obj.include_replies + reply_found = True + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) + if reply_found: + assert sample_user_obj.include_replies return mock @@ -648,33 +719,41 @@ def test_hashtags(sample_users, global_mock): sample_user_obj = User( user_item, users['config'], os.getcwd() ) - if sample_user_obj.hashtags: - tweets_v2 = sample_user_obj._get_tweets("v2") - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - for tweet in tweets_to_post['data']: - tweet_hashtags = tweet["entities"]["hashtags"] - i = 0 - while i < len(tweet_hashtags): - if ( - tweet_hashtags[i]["tag"] - in sample_user_obj.hashtags - ): - match = True - break - i += 1 - else: - match = False - - assert match - - # Clean up - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] + for t_user in sample_user_obj.twitter_username: + if sample_user_obj.hashtags: + tweets_v2 = sample_user_obj._get_tweets( + "v2", t_user=t_user ) - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - os.remove(file_path) + tweets_to_post = sample_user_obj.process_tweets( + tweets_v2 + ) + for tweet in tweets_to_post['data']: + tweet_hashtags = tweet["entities"]["hashtags"] + i = 0 + while i < len(tweet_hashtags): + if ( + tweet_hashtags[i]["tag"] + in sample_user_obj.hashtags + ): + match = True + break + i += 1 + else: + match = False + + assert match + + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join( + tweet_folder, file + ) + if os.path.isfile(file_path): + os.remove(file_path) return mock, sample_user @@ -693,74 +772,144 @@ def test_nitter_instances(sample_users, mock_request, global_mock): sample_user_obj = User( user_item, users_nitter_net['config'], os.getcwd() ) - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets( + "v2", t_user=t_user + ) + sample_data = mock_request['sample_data'] + assert tweets_v2 == sample_data['tweets_v2'] + tweet = sample_user_obj._get_tweets( + "v1.1", test_user.pinned + ) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - for tweet in tweets_to_post['data']: - if sample_user_obj.signature: - sample_user_obj.post_pleroma( - ( - tweet["id"], - tweet["text"], - tweet["created_at"], - ), None, False - ) - history = mock.request_history - assert nitter_instance in parse.unquote( - history[-1].text - ) + for tweet in tweets_to_post['data']: + if sample_user_obj.signature: + sample_user_obj.post_pleroma( + ( + tweet["id"], + tweet["text"], + tweet["created_at"], + ), None, False + ) + history = mock.request_history + assert nitter_instance in parse.unquote( + history[-1].text + ) # Clean up - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] - ) - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - os.remove(file_path) + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) for user_item in users_nitter['user_dict']: nitter_instance = users_nitter['config']['nitter_base_url'] sample_user_obj = User( user_item, users_nitter['config'], os.getcwd() ) - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets( + "v2", t_user=t_user + ) + sample_data = mock_request['sample_data'] + assert tweets_v2 == sample_data['tweets_v2'] + tweet = sample_user_obj._get_tweets( + "v1.1", test_user.pinned + ) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - for tweet in tweets_to_post['data']: - if sample_user_obj.signature: - sample_user_obj.post_pleroma( - ( - tweet["id"], - tweet["text"], - tweet["created_at"] - ), None, False - ) - history = mock.request_history - assert nitter_instance in parse.unquote( - history[-1].text + for tweet in tweets_to_post['data']: + if sample_user_obj.signature: + sample_user_obj.post_pleroma( + ( + tweet["id"], + tweet["text"], + tweet["created_at"] + ), None, False + ) + history = mock.request_history + assert nitter_instance in parse.unquote( + history[-1].text + ) + + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) + return mock, sample_user - # Clean up - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] + +def test_invidious_instances(sample_users, mock_request, global_mock): + test_user = UserTemplate() + for sample_user in sample_users: + with global_mock as mock: + users_invidious = get_config_users( + 'config_invidious_different_instance.yml' + ) + for user_item in users_invidious['user_dict']: + invidious_instance = user_item['invidious_base_url'] + sample_user_obj = User( + user_item, users_invidious['config'], os.getcwd() + ) + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets( + "v2", t_user=t_user ) - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - os.remove(file_path) + sample_data = mock_request['sample_data'] + assert tweets_v2 == sample_data['tweets_v2'] + tweet = sample_user_obj._get_tweets( + "v1.1", test_user.pinned + ) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] + + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + + for tweet in tweets_to_post['data']: + if sample_user_obj.signature: + sample_user_obj.post_pleroma( + ( + tweet["id"], + tweet["text"], + tweet["created_at"] + ), None, False + ) + history = mock.request_history + parsed = parse.unquote( + history[-1].text + ) + assert "https://youtube.com" not in parsed + if "/watch?v=dQw4w9WgXcQ" in parsed: + assert invidious_instance in parsed + + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) return mock, sample_user @@ -775,90 +924,290 @@ def test_original_date(sample_users, mock_request, global_mock): sample_user_obj = User( user_item, users_date['config'], os.getcwd() ) - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets( + "v2", t_user=t_user + ) + sample_data = mock_request['sample_data'] + assert tweets_v2 == sample_data['tweets_v2'] + tweet = sample_user_obj._get_tweets( + "v1.1", test_user.pinned + ) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - for tweet in tweets_to_post['data']: - if sample_user_obj.signature: - sample_user_obj.post_pleroma( - ( - tweet["id"], - tweet["text"], - tweet["created_at"], - ), None, False - ) - history = mock.request_history - tweet_date = tweet["created_at"] - date_format = sample_user_obj.original_date_format - date = datetime.strftime( - datetime.strptime( - tweet_date, "%Y-%m-%dT%H:%M:%S.000Z" - ), - date_format, - ) - assert f"[{date}]" in parse.unquote( - history[-1].text.replace("+", " ") - ) + for tweet in tweets_to_post['data']: + if sample_user_obj.signature: + sample_user_obj.post_pleroma( + ( + tweet["id"], + tweet["text"], + tweet["created_at"], + ), None, False + ) + history = mock.request_history + tweet_date = tweet["created_at"] + date_format = sample_user_obj.original_date_format + date = datetime.strftime( + datetime.strptime( + tweet_date, "%Y-%m-%dT%H:%M:%S.000Z" + ), + date_format, + ) + assert f"[{date}]" in parse.unquote( + history[-1].text.replace("+", " ") + ) # Clean up - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] - ) - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - os.remove(file_path) + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) for user_item in users_no_date['user_dict']: sample_user_obj = User( user_item, users_date['config'], os.getcwd() ) - tweets_v2 = sample_user_obj._get_tweets("v2") - assert tweets_v2 == mock_request['sample_data']['tweets_v2'] - tweet = sample_user_obj._get_tweets("v1.1", test_user.pinned) - assert tweet == mock_request['sample_data']['tweet'] - tweets = sample_user_obj._get_tweets("v1.1") - assert tweets == mock_request['sample_data']['tweets_v1'] + for t_user in sample_user_obj.twitter_username: + tweets_v2 = sample_user_obj._get_tweets( + "v2", t_user=t_user + ) + sample_data = mock_request['sample_data'] + assert tweets_v2 == sample_data['tweets_v2'] + tweet = sample_user_obj._get_tweets( + "v1.1", test_user.pinned + ) + assert tweet == mock_request['sample_data']['tweet'] + tweets = sample_user_obj._get_tweets("v1.1") + assert tweets == mock_request['sample_data']['tweets_v1'] - tweets_to_post = sample_user_obj.process_tweets(tweets_v2) + tweets_to_post = sample_user_obj.process_tweets(tweets_v2) - for tweet in tweets_to_post['data']: - if sample_user_obj.signature: - sample_user_obj.post_pleroma( - ( - tweet["id"], - tweet["text"], - tweet["created_at"], - ), None, False - ) - history = mock.request_history - tweet_date = tweet["created_at"] - date_format = sample_user_obj.original_date_format - date = datetime.strftime( - datetime.strptime( - tweet_date, "%Y-%m-%dT%H:%M:%S.000Z" - ), - date_format, + for tweet in tweets_to_post['data']: + if sample_user_obj.signature: + sample_user_obj.post_pleroma( + ( + tweet["id"], + tweet["text"], + tweet["created_at"], + ), None, False + ) + history = mock.request_history + date_format = sample_user_obj.original_date_format + date = datetime.strftime( + datetime.strptime( + tweet_date, "%Y-%m-%dT%H:%M:%S.000Z" + ), + date_format, + ) + assert f"[{date}]" not in parse.unquote( + history[-1].text.replace("+", " ") + ) + + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] ) - assert f"[{date}]" not in parse.unquote( - history[-1].text.replace("+", " ") + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) + return mock, sample_user + + +def test_tweet_order(sample_users, mock_request, global_mock): + test_user = UserTemplate() + for sample_user in sample_users: + with global_mock as mock: + users = get_config_users('config.yml') + + for user_item in users['user_dict']: + sample_user_obj = User( + user_item, users['config'], os.getcwd() + ) + for t_user in sample_user_obj.twitter_username: + tweets = sample_user_obj._get_tweets( + "v2", t_user=t_user + ) + assert tweets == mock_request['sample_data']['tweets_v2'] + + # Test start sample data order + tweets_to_post = sample_user_obj.process_tweets(tweets) + tw_z = zip(tweets["data"], tweets_to_post["data"]) + + for prev, (f, b), nxt in previous_and_next(tw_z): + assert f["created_at"] == b["created_at"] + # prev and nxt are not None (start or end of list) + if all((prev, nxt)): + (prevf, prevb) = prev + (nxtf, nxtb) = nxt + assert prevf["created_at"] > f["created_at"] + assert f["created_at"] > nxtf["created_at"] + + assert prevb["created_at"] > b["created_at"] + assert b["created_at"] > nxtb["created_at"] + + # Test mp reverse order + tweets = sample_user_obj._get_tweets( + "v2", t_user=t_user + ) + assert tweets == mock_request['sample_data']['tweets_v2'] + tweets["data"].reverse() + cores = mp.cpu_count() + threads = round(cores / 2 if cores > 4 else 4) + tweets_to_post = process_parallel( + tweets, sample_user_obj, threads + ) + tw_z = zip(tweets["data"], tweets_to_post["data"]) + for prev, (f, b), nxt in previous_and_next(tw_z): + assert f["created_at"] == b["created_at"] + # prev and nxt are not None (start or end of list) + if all((prev, nxt)): + (prevf, prevb) = prev + (nxtf, nxtb) = nxt + assert prevf["created_at"] < f["created_at"] + assert f["created_at"] < nxtf["created_at"] + + assert prevb["created_at"] < b["created_at"] + assert b["created_at"] < nxtb["created_at"] + for tweet in tweets_to_post["data"]: + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) + return test_user, sample_user, mock + +def test_keep_media_links(sample_users, mock_request, global_mock): + test_user = UserTemplate() + for sample_user in sample_users: + with global_mock as mock: + users_keep = get_config_users('config_keep_media_links.yml') + users_no_keep = get_config_users('config_no_keep_media_links.yml') + for user_item in users_keep['user_dict']: + sample_user_obj = User( + user_item, users_keep['config'], os.getcwd() + ) + tweet_v2 = sample_user_obj._get_tweets( + "v2", sample_user_obj.pinned_tweet_id + ) + assert tweet_v2 == mock_request['sample_data']['pinned_tweet'] + tweet_v2["data"]["text"] = sample_user_obj._expand_urls( + tweet_v2["data"] + ) + regex = r"\bhttps?:\/\/twitter.com\/+[^\/:]+\/.*?" \ + r"(photo|video)\/\d*\b" + matches = [] + match = re.search(regex, tweet_v2["data"]["text"]) + if match: + matches.append(match.group()) + + tweets_to_post = sample_user_obj.process_tweets( + {"data": [tweet_v2["data"]], + "includes": tweet_v2["includes"]} + ) + for m in matches: + for tweet in tweets_to_post["data"]: + if sample_user_obj.keep_media_links: + assert m in tweet["text"] # Clean up - tweet_folder = os.path.join( - sample_user_obj.tweets_temp_path, tweet["id"] - ) - for file in os.listdir(tweet_folder): - file_path = os.path.join(tweet_folder, file) - if os.path.isfile(file_path): - os.remove(file_path) - return mock, sample_user + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) + for sample_user in sample_users: + with global_mock as mock: + users_keep = get_config_users('config_keep_media_links.yml') + users_no_keep = get_config_users('config_no_keep_media_links.yml') + for user_item in users_keep['user_dict']: + sample_user_obj = User( + user_item, users_keep['config'], os.getcwd() + ) + tweet_v2 = sample_user_obj._get_tweets( + "v2", sample_user_obj.pinned_tweet_id + ) + assert tweet_v2 == mock_request['sample_data']['pinned_tweet'] + tweet_v2["data"]["text"] = sample_user_obj._expand_urls( + tweet_v2["data"] + ) + regex = r"\bhttps?:\/\/twitter.com\/+[^\/:]+\/.*?" \ + r"(photo|video)\/\d*\b" + matches = [] + match = re.search(regex, tweet_v2["data"]["text"]) + if match: + matches.append(match.group()) + + tweets_to_post = sample_user_obj.process_tweets( + {"data": [tweet_v2["data"]], + "includes": tweet_v2["includes"]} + ) + for m in matches: + for tweet in tweets_to_post["data"]: + if sample_user_obj.keep_media_links: + assert m in tweet["text"] + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) + for user_item in users_no_keep['user_dict']: + sample_user_obj = User( + user_item, users_no_keep['config'], os.getcwd() + ) + tweet_v2 = sample_user_obj._get_tweets( + "v2", sample_user_obj.pinned_tweet_id + ) + assert tweet_v2 == mock_request['sample_data']['pinned_tweet'] + tweet_v2["data"]["text"] = sample_user_obj._expand_urls( + tweet_v2["data"] + ) + regex = r"https:\/\/twitter\.com\/.*\/status\/.*\/(" \ + r"photo|video)\/\d" + matches = [] + match = re.search(regex, tweet_v2["data"]["text"]) + if match: + matches.append(match.group()) + + tweets_to_post = sample_user_obj.process_tweets( + {"data": [tweet_v2["data"]], + "includes": tweet_v2["includes"]} + ) + for m in matches: + for tweet in tweets_to_post["data"]: + if not sample_user_obj.keep_media_links: + assert m not in tweet["text"] + # Clean up + tweet_folder = os.path.join( + sample_user_obj.tweets_temp_path, tweet["id"] + ) + if os.path.isdir(tweet_folder): + for file in os.listdir(tweet_folder): + file_path = os.path.join(tweet_folder, file) + if os.path.isfile(file_path): + os.remove(file_path) + + return mock, sample_user, test_user def test__process_polls_with_media(sample_users): @@ -937,6 +1286,24 @@ def test_main(rootdir, global_mock, sample_users, monkeypatch): monkeypatch.setattr('builtins.input', lambda: "2020-12-30") with patch.object(sys, 'argv', ['']): assert cli.main() == 0 + + sample_data_dir = os.path.join(test_files_dir, 'sample_data') + media_dir = os.path.join(sample_data_dir, 'media') + archive = os.path.join(media_dir, 'twitter-archive.zip') + monkeypatch.setattr('builtins.input', lambda: "2021-12-11") + with patch.object(sys, 'argv', ['', '--archive', archive]): + assert cli.main() == 0 + + monkeypatch.setattr('builtins.input', lambda: "") + with patch.object(sys, 'argv', ['', '--archive', archive]): + assert cli.main() == 0 + + monkeypatch.setattr('builtins.input', lambda: "2021-12-13") + with patch.object( + sys, 'argv', ['', '--archive', archive, '--forceDate'] + ): + assert cli.main() == 0 + # Test main() is called correctly when name equals __main__ with patch.object(cli, "main", return_value=42): with patch.object(cli, "__name__", "__main__"): @@ -965,16 +1332,22 @@ def test_main(rootdir, global_mock, sample_users, monkeypatch): shutil.copy(backup_config, prev_config) for sample_user in sample_users: sample_user_obj = sample_user['user_obj'] - pinned_path = os.path.join(os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id.txt') - pinned_pleroma = os.path.join(os.getcwd(), - 'users', - sample_user_obj.twitter_username, - 'pinned_id_pleroma.txt') - if os.path.isfile(pinned_path): - os.remove(pinned_path) - if os.path.isfile(pinned_pleroma): - os.remove(pinned_pleroma) + for t_user in sample_user_obj.twitter_username: + idx = sample_user_obj.twitter_username.index(t_user) + pinned_path = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id.txt' + ) + pinned_pleroma = os.path.join( + os.getcwd(), + 'users', + sample_user_obj.twitter_username[idx], + 'pinned_id_pleroma.txt' + ) + if os.path.isfile(pinned_path): + os.remove(pinned_path) + if os.path.isfile(pinned_pleroma): + os.remove(pinned_pleroma) return g_mock