merge(release/1.2.3): enable challenge solver video record optional

MonitorrentInstaller/MonitorrentInstaller/Product.wxs

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
- <Product Id="*" Name="Monitorrent" Language="1033" Version="1.2.2.100" Manufacturer="Monitorrent Team" UpgradeCode="dd4cf505-1e44-4311-a8f2-efcf097175a7">
+ <Product Id="*" Name="Monitorrent" Language="1033" Version="1.2.3.100" Manufacturer="Monitorrent Team" UpgradeCode="dd4cf505-1e44-4311-a8f2-efcf097175a7">
  <Package InstallerVersion="200" Compressed="yes" InstallScope="perMachine" />
 
  <MajorUpgrade DowngradeErrorMessage="A newer version of [ProductName] is already installed." AllowSameVersionUpgrades="yes"/>

README.md

@@ -56,14 +56,14 @@ Multiplatform image (`arm64` and `amd64` (`arm` is not supported anymore, until
 [https://hub.docker.com/r/werwolfby/monitorrent/](https://hub.docker.com/r/werwolfby/monitorrent/)
 
 ### Windows Installer:
-https://github.com/werwolfby/monitorrent/releases/download/1.2.2/MonitorrentInstaller-1.2.2.msi
+https://github.com/werwolfby/monitorrent/releases/download/1.2.3/MonitorrentInstaller-1.2.3.msi
 
 ### Manual Install
 
 Requirements:
  - Python 3.9+ and pip
 
-Download latest build: https://github.com/werwolfby/monitorrent/releases/download/1.2.2/monitorrent-1.2.2.zip
+Download latest build: https://github.com/werwolfby/monitorrent/releases/download/1.2.3/monitorrent-1.2.3.zip
 Extract into **monitorent** folder
  * pip install -r requirements.txt
  * playwright --with-deps install firefox

monitorrent/__init__.py

@@ -1 +1 @@
-__version__ = '1.2.2'
+__version__ = '1.2.3'

monitorrent/plugin_managers.py

@@ -61,22 +61,22 @@ def get_settings(self, name):
 
  def set_settings(self, name, settings):
  tracker = self.get_tracker(name)
+ tracker_settings = self.settings_manager.tracker_settings
+ tracker.init(tracker_settings)
  if hasattr(tracker, 'update_credentials'):
  tracker.update_credentials(settings)
  return True
  return False
 
  def check_connection(self, name):
  tracker = self.get_tracker(name)
- if not isinstance(tracker, WithCredentialsMixin):
- return False
  tracker_settings = self.settings_manager.tracker_settings
- # get_tracker returns PluginTrackerBase
- # noinspection PyUnresolvedReferences
  tracker.init(tracker_settings)
+ if not isinstance(tracker, WithCredentialsMixin):
+ return False
  return tracker.verify()
 
- def get_tracker(self, name):
+ def get_tracker(self, name: str) -> TrackerPluginBase:
  if name not in self.trackers:
  raise KeyError('Tracker {} not found'.format(name))
  return self.trackers[name]

monitorrent/plugins/trackers/__init__.py

@@ -29,10 +29,33 @@
 from future.utils import with_metaclass
 
 
+class CloudflareChallengeSolverSettings(object):
+ def __init__(self, debug, timeout, record_video, record_har, keep_records):
+ self.debug = debug
+ self.timeout = timeout
+ self.record_video = record_video
+ self.record_har = record_har
+ self.keep_records = keep_records
+
+ def get_new_context_kwargs(self, video_folder):
+ if not self.debug or video_folder is None:
+ return {}
+
+ kwargs = {}
+ if self.record_video:
+ kwargs['record_video_dir'] = video_folder
+ kwargs['record_video_size'] = {"width": 1024, "height": 768}
+ if self.record_har:
+ kwargs['record_har_path'] = path.join(video_folder, 'challenge.har')
+
+ return kwargs
+
+
 class TrackerSettings(object):
- def __init__(self, requests_timeout, proxies):
+ def __init__(self, requests_timeout, proxies, cloudflare_challenge_solver_settings):
  self.requests_timeout = requests_timeout
  self.proxies = proxies
+ self.cloudflare_challenge_solver_settings = cloudflare_challenge_solver_settings
 
  def get_requests_kwargs(self):
  return {'timeout': self.requests_timeout, 'proxies': self.proxies}
@@ -361,7 +384,22 @@ def _execute_login(self, engine):
  return True
 
 
-def extract_cloudflare_credentials_and_headers(url: str, headers: dict, cookies: dict, timeout: int = 120000):
+def update_headers_and_cookies_mixin(self, url):
+ if not hasattr(self, 'headers') or not hasattr(self, 'cookies') or not hasattr(self, 'headers_cookies_updater'):
+ raise Exception('headers, cookies and headers_cookies_updater should be defined in object')
+
+ headers, cookies = extract_cloudflare_credentials_and_headers(url, self.headers, self.cookies,
+ self.tracker_settings.cloudflare_challenge_solver_settings)
+ if headers != self.headers or cookies != self.cookies:
+ self.headers = headers
+ self.cookies = cookies
+
+ self.headers_cookies_updater(self.headers, self.cookies)
+
+ return headers, cookies
+
+
+def extract_cloudflare_credentials_and_headers(url: str, headers: dict, cookies: dict, settings: CloudflareChallengeSolverSettings):
  scrapper = cloudscraper.create_scraper()
 
  try:
@@ -372,16 +410,14 @@ def extract_cloudflare_credentials_and_headers(url: str, headers: dict, cookies:
  # If page doesn't have cloudflare, don't send new cookies and headers
  return headers, cookies
  except CloudflareException:
- return asyncio.run(solve_challenge(url, timeout=timeout))
-
+ return asyncio.run(solve_challenge(url, settings))
 
-async def solve_challenge(url, timeout=120000):
- video_folder = path.join('webapp', 'challenges', datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
- os.makedirs(video_folder, exist_ok=True)
 
- # keep only 3 last challenges, delete others
- for challenge_folder in sorted(glob.glob(path.join('webapp', 'challenges', '*')), reverse=True, key=path.getctime)[3:]:
- shutil.rmtree(challenge_folder)
+async def solve_challenge(url, settings: CloudflareChallengeSolverSettings):
+ video_folder = None
+ if settings.debug:
+ video_folder = path.join('webapp', 'challenges', datetime.now().strftime("%Y-%m-%d_%H-%M-%S"))
+ os.makedirs(video_folder, exist_ok=True)
 
  browser_launch_kwargs = get_browser_launch_kwargs()
  ws_endpoint = browser_launch_kwargs.pop('ws_endpoint', '')
@@ -391,7 +427,7 @@ async def solve_challenge(url, timeout=120000):
  browser = await p.firefox.connect(ws_endpoint=ws_endpoint)
  else:
  browser = await p.firefox.launch(**browser_launch_kwargs)
- context = await browser.new_context(record_har_path=path.join(video_folder, 'challenge.har'), record_video_dir=video_folder, record_video_size={"width": 1024, "height": 768})
+ context = await browser.new_context(**settings.get_new_context_kwargs(video_folder))
  try:
  page = await context.new_page()
 
@@ -407,9 +443,9 @@ async def on_request(req):
  await page.goto(url)
 
  features = [
- asyncio.create_task(page.locator('input[type="button"]').click(timeout=timeout)),
- asyncio.create_task(page.frame_locator("iframe").locator("input").click(timeout=timeout)),
- asyncio.create_task(page.wait_for_selector('.left-side > .menu', timeout=timeout)),
+ asyncio.create_task(page.locator('input[type="button"]').click(timeout=settings.timeout)),
+ asyncio.create_task(page.frame_locator("iframe").locator("input").click(timeout=settings.timeout)),
+ asyncio.create_task(page.wait_for_selector('.left-side > .menu', timeout=settings.timeout)),
  ]
 
  done, rest = await asyncio.wait(features, return_when=asyncio.FIRST_COMPLETED)
@@ -438,6 +474,10 @@ async def on_request(req):
  await context.close()
  await browser.close()
 
+ # keep only settings.keep_records last challenges, delete others
+ for challenge_folder in sorted(glob.glob(path.join('webapp', 'challenges', '*')), reverse=True, key=path.getctime)[settings.keep_records:]:
+ shutil.rmtree(challenge_folder)
+
 
 async def wait_for_iframe_input(page, timeout=120000):
  await page.frame_locator("iframe").locator("input").click(timeout=timeout)

monitorrent/plugins/trackers/lostfilm.py

@@ -16,8 +16,8 @@
 from monitorrent.utils.downloader import download
 from monitorrent.plugins import Topic
 from monitorrent.plugins.status import Status
-from monitorrent.plugins.trackers import TrackerPluginBase, WithCredentialsMixin, LoginResult, \
- extract_cloudflare_credentials_and_headers
+from monitorrent.plugins.trackers import TrackerPluginBase, WithCredentialsMixin, LoginResult, TrackerSettings, \
+ update_headers_and_cookies_mixin
 from monitorrent.plugins.clients import TopicSettings
 import html
 
@@ -454,7 +454,7 @@ def __init__(self, quality, download_url):
 
 
 class LostFilmTVTracker(object):
- tracker_settings = None
+ tracker_settings: TrackerSettings = None
  _season_title_info = re.compile(u'^(?P<season>\d+)(\.(?P<season_fraction>\d+))?\s+сезон' +
  u'(\s+((\d+)-)?(?P<episode>\d+)\s+серия)?$')
  _follow_show_re = re.compile(r'^FollowSerial\((?P<cat>\d+)\)$', re.UNICODE)
@@ -481,7 +481,7 @@ def setup(self, session=None, headers=None, cookies=None):
  def login(self, email, password, headers=None, cookies=None):
  self.headers = headers or {}
  self.cookies = cookies or {}
- headers, cookies = self._update_headers_and_cookies("https://" + self.netloc)
+ headers, cookies = update_headers_and_cookies_mixin(self, "https://" + self.netloc)
 
  params = {"act": "users", "type": "login", "mail": email, "pass": password, "rem": 1, "need_captcha": "", "captcha": ""}
  response = requests.post("https://www.lostfilm.tv/ajaxik.users.php", params, headers=headers, cookies=cookies)
@@ -499,7 +499,7 @@ def verify(self):
  if not cookies:
  return False
  my_settings_url = 'https://www.lostfilm.tv/my_settings'
- self._update_headers_and_cookies(my_settings_url)
+ update_headers_and_cookies_mixin(self, my_settings_url)
  r1 = requests.get(my_settings_url, headers=self.headers, cookies=self.get_cookies(),
  **self.tracker_settings.get_requests_kwargs())
  return r1.url == my_settings_url and '<meta http-equiv="refresh" content="0; url=/">' not in r1.text
@@ -521,7 +521,7 @@ def parse_url(self, url, parse_series=False):
  if url is None:
  return None
 
- self._update_headers_and_cookies(url)
+ update_headers_and_cookies_mixin(self, url)
 
  response = requests.get(url, headers=self.headers, cookies=self.get_cookies(), allow_redirects=False,
  **self.tracker_settings.get_requests_kwargs())
@@ -596,7 +596,7 @@ def parse_download(table):
 
  return LostFileDownloadInfo(LostFilmQuality.parse(quality), download_url)
 
- self._update_headers_and_cookies(url)
+ update_headers_and_cookies_mixin(self, url)
 
  download_redirect_url = self.download_url_pattern.format(cat=cat, season=season, episode=episode)
  session = requests.session()
@@ -613,16 +613,6 @@ def parse_download(table):
  soup = get_soup(download_page.text)
  return list(map(parse_download, soup.find_all('div', class_='inner-box--item')))
 
- def _update_headers_and_cookies(self, url):
- headers, cookies = extract_cloudflare_credentials_and_headers(url, self.headers, self.cookies)
- if headers != self.headers or cookies != self.cookies:
- self.headers = headers
- self.cookies = cookies
-
- self.headers_cookies_updater(self.headers, self.cookies)
-
- return headers, cookies
-
 
 class LostFilmPlugin(WithCredentialsMixin, TrackerPluginBase):
  credentials_class = LostFilmTVCredentials
@@ -759,9 +749,7 @@ def login(self):
  if not username or not password:
  return LoginResult.CredentialsNotSpecified
  try:
- self.tracker.login(username, password,
- headers or self.tracker.headers,
- cookies or self.tracker.cookies)
+ self.tracker.login(username, password, headers, cookies)
  with DBSession() as db:
  cred = db.query(self.credentials_class).first()
  cred.session = self.tracker.session

monitorrent/plugins/trackers/rutracker.py

@@ -11,7 +11,7 @@
 from monitorrent.plugin_managers import register_plugin
 from monitorrent.utils.soup import get_soup
 from monitorrent.plugins.trackers import TrackerPluginBase, WithCredentialsMixin, ExecuteWithHashChangeMixin, \
- LoginResult, extract_cloudflare_credentials_and_headers
+ LoginResult, TrackerSettings, update_headers_and_cookies_mixin
 
 PLUGIN_NAME = 'rutracker.org'
 
@@ -67,7 +67,7 @@ def get_current_version(engine):
 
 
 class RutrackerTracker(object):
- tracker_settings = None
+ tracker_settings: TrackerSettings = None
  login_url = "https://rutracker.org/forum/login.php"
  profile_page = "https://rutracker.org/forum/privmsg.php?folder=inbox"
  _regex = re.compile(six.text_type(r'^https?://w*\.*rutracker.org/forum/viewtopic.php\?t=(\d+)(/.*)?$'))
@@ -109,7 +109,7 @@ def parse_url(self, url):
  def login(self, username, password, headers=None, cookies=None):
  self.headers = headers
  self.cookies = cookies
- self._update_headers_and_cookies("https://rutracker.org/forum/index.php")
+ update_headers_and_cookies_mixin(self, "https://rutracker.org/forum/index.php")
 
  username_q = username.encode('windows-1251')
  password_q = password.encode('windows-1251')
@@ -164,16 +164,6 @@ def get_download_url(self, url):
 
  return "https://rutracker.org/forum/dl.php?t=" + id
 
- def _update_headers_and_cookies(self, url):
- headers, cookies = extract_cloudflare_credentials_and_headers(url, self.headers, self.cookies)
- if headers != self.headers or cookies != self.cookies:
- self.headers = headers
- self.cookies = cookies
-
- self.headers_cookies_updater(self.headers, self.cookies)
-
- return headers, cookies
-
 
 class RutrackerPlugin(WithCredentialsMixin, ExecuteWithHashChangeMixin, TrackerPluginBase):
  tracker = RutrackerTracker()

monitorrent/rest/challenge_logs.py

@@ -17,16 +17,17 @@ def on_get(self, req, resp):
  # scan folder webapp/challenges
  challenges_folder = path.join('webapp', 'challenges')
  challenges = []
- for folder in sorted(os.listdir(challenges_folder), reverse=True):
- challenge_folder = path.join(challenges_folder, folder)
- if not os.path.isdir(challenge_folder):
- continue
- challenge = {
- 'folder': folder,
- 'video': self._get_video(challenge_folder),
- 'har': self._get_har(challenge_folder),
- }
- challenges.append(challenge)
+ if os.path.exists(challenges_folder):
+ for folder in sorted(os.listdir(challenges_folder), reverse=True):
+ challenge_folder = path.join(challenges_folder, folder)
+ if not os.path.isdir(challenge_folder):
+ continue
+ challenge = {
+ 'folder': folder,
+ 'video': self._get_video(challenge_folder),
+ 'har': self._get_har(challenge_folder),
+ }
+ challenges.append(challenge)
 
  resp.json = challenges
 

monitorrent/rest/settings_cloudflare_challenge_solver.py

@@ -0,0 +1,59 @@
+import falcon
+import six
+from monitorrent.settings_manager import SettingsManager
+from monitorrent.new_version_checker import NewVersionChecker
+
+
+# noinspection PyUnusedLocal
+class SettingsCloudflareChallengeSolver(object):
+ def __init__(self, settings_manager):
+ """
+ :type settings_manager: SettingsManager
+ """
+ self.settings_manager = settings_manager
+
+ def on_get(self, req, resp):
+ resp.json = {
+ 'debug': self.settings_manager.cloudflare_challenge_solver_debug,
+ 'record_video': self.settings_manager.cloudflare_challenge_solver_record_video,
+ 'record_har': self.settings_manager.cloudflare_challenge_solver_record_har,
+ 'keep_records': self.settings_manager.cloudflare_challenge_solver_keep_records,
+ }
+
+ def on_patch(self, req, resp):
+ if req.json is None or len(req.json) == 0:
+ raise falcon.HTTPBadRequest('BodyRequired', 'Expecting not empty JSON body')
+
+ debug = req.json.get('debug')
+ if debug is not None and not isinstance(debug, bool):
+ raise falcon.HTTPBadRequest('WrongValue', '"debug" have to be bool')
+
+ record_video = req.json.get('record_video')
+ if record_video is not None and not isinstance(record_video, bool):
+ raise falcon.HTTPBadRequest('WrongValue', '"record_video" have to be bool')
+
+ record_har = req.json.get('record_har')
+ if record_har is not None and not isinstance(record_har, bool):
+ raise falcon.HTTPBadRequest('WrongValue', '"record_har" have to be bool')
+
+ keep_records = req.json.get('keep_records')
+ if keep_records is not None and not isinstance(keep_records, int):
+ raise falcon.HTTPBadRequest('WrongValue', '"keep_records" have to be int')
+
+ if debug is not None:
+ if self.settings_manager.cloudflare_challenge_solver_debug != debug:
+ self.settings_manager.cloudflare_challenge_solver_debug = debug
+
+ if record_video is not None:
+ if self.settings_manager.cloudflare_challenge_solver_record_video != record_video:
+ self.settings_manager.cloudflare_challenge_solver_record_video = record_video
+
+ if record_har is not None:
+ if self.settings_manager.cloudflare_challenge_solver_record_har != record_har:
+ self.settings_manager.cloudflare_challenge_solver_record_har = record_har
+
+ if keep_records is not None:
+ if self.settings_manager.cloudflare_challenge_solver_keep_records != keep_records:
+ self.settings_manager.cloudflare_challenge_solver_keep_records = keep_records
+
+ resp.status = falcon.HTTP_NO_CONTENT