Merge branch 'develop'

.gitattributes

@@ -0,0 +1,2 @@
+# downloaded htmls for testing purpose only
+tests/httprety/* linguist-vendored

.travis.yml

@@ -5,7 +5,7 @@ python:
 install:
 - pip install -r requirements-dev.txt
 - pip install codecov
-script: coverage run -m unittest discover -s monitorrent/tests
+script: coverage run -m unittest discover -s tests
 after_success: 
 - coveralls
 - codecov

MonitorrentInstaller/MonitorrentInstaller/Product.wxs

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Wix xmlns="http://schemas.microsoft.com/wix/2006/wi">
- <Product Id="*" Name="Monitorrent" Language="1033" Version="1.0.1.11" Manufacturer="Monitorrent Team" UpgradeCode="dd4cf505-1e44-4311-a8f2-efcf097175a7">
+ <Product Id="*" Name="Monitorrent" Language="1033" Version="1.1.0.30" Manufacturer="Monitorrent Team" UpgradeCode="dd4cf505-1e44-4311-a8f2-efcf097175a7">
  <Package InstallerVersion="200" Compressed="yes" InstallScope="perMachine" />
 
  <MajorUpgrade DowngradeErrorMessage="A newer version of [ProductName] is already installed." AllowSameVersionUpgrades="yes"/>

appveyor.yml

@@ -42,18 +42,21 @@ install:
  - "python -m pip install --disable-pip-version-check --user --upgrade pip"
 
  - ps: >-
- $lxmlName = $env:LXML_NAME
+ $lxmlVersion = '3.6.4'
 
  if($env:PYTHON_VERSION -eq "3.5.x")
-
  {
-
- wget http://www.edna-site.org/pub/wheelhouse/lxml-3.6.0-cp35-cp35m-win32.whl -OutFile "$env:APPVEYOR_BUILD_FOLDER\lxml-3.6.0-cp35-cp35m-win32.whl"
-
- & pip install "$env:APPVEYOR_BUILD_FOLDER\lxml-3.6.0-cp35-cp35m-win32.whl"
-
+ $lxmlName = "lxml-$lxmlVersion-cp35-cp35m-win32.whl"
  }
+ else
+ {
+ $lxmlName = "lxml-$lxmlVersion-cp27-cp27m-win32.whl"
+ }
+
+ wget "http://www.edna-site.org/pub/wheelhouse/$lxmlName" -OutFile "$env:APPVEYOR_BUILD_FOLDER\$lxmlName"
 
+ & pip install "$env:APPVEYOR_BUILD_FOLDER\$lxmlName"
+ 
  & pip install codecov
 
  #workaround for slow lxml download
@@ -88,7 +91,7 @@ on_success:
 
 test_script:
  # Run the project tests
- - coverage run -m unittest discover -s %APPVEYOR_BUILD_FOLDER%/monitorrent/tests
+ - coverage run -m unittest discover -s %APPVEYOR_BUILD_FOLDER%/tests
 
 notifications:
  - provider: Slack

cover.cmd

@@ -1,3 +1,3 @@
-coverage run -m unittest discover -s monitorrent/tests
+coverage run -m unittest discover -s tests
 coverage html
 start htmlcov\index.html

gulpfile.js

@@ -11,13 +11,18 @@ var zip = require('gulp-zip');
 var rename = require('gulp-rename');
 var ngAnnotate = require('gulp-ng-annotate');
 var uglify = require('gulp-uglify');
+var git = require('git-rev');
+var htmlmin = require('gulp-htmlmin');
+var templateCache = require('gulp-angular-templatecache');
+var addStream = require('add-stream');
 
 var pkg = require('./package.json');
 
 var paths = {
  scripts: ['./src/**/*.js'],
  index_pages: ['./src/*.html'],
- statics: ['./src/**/*.html', './src/**/*.svg', './src/**/*.png', './src/favicon.ico', '!./src/*.html'],
+ templates: ['./src/**/*.html', '!./src/*.html'],
+ statics: ['./src/**/*.svg', './src/**/*.png', './src/favicon.ico'],
  styles: ['./src/**/*.less'],
  dest: 'webapp',
  release: 'dist'
@@ -39,21 +44,39 @@ gulp.task('jshint', function () {
  .pipe(jshint.reporter('default'));
 });
 
-gulp.task('concat', function () {
- return gulp.src(paths.scripts)
- .pipe(sourcemaps.init())
- .pipe(ngAnnotate())
- .pipe(concat(pkg.name + '.js'))
- .pipe(uglify())
- .pipe(sourcemaps.write('.'))
- .pipe(gulp.dest(path.join(paths.dest, 'scripts')));
+gulp.task('concat', function (cb) {
+ git.long(function (rev) {
+ var stream = gulp.src(paths.scripts)
+ .pipe(addStream.obj(prepareTemplates()))
+ .pipe(sourcemaps.init())
+ .pipe(ngAnnotate())
+ .pipe(concat(pkg.name + '.js'))
+ .pipe(preprocess({
+ context: { VERSION: pkg.version, COMMIT_HASH: rev }
+ }))
+ .pipe(uglify())
+ .pipe(sourcemaps.write('.'))
+ .pipe(gulp.dest(path.join(paths.dest, 'scripts')))
+ .on('end', cb);
+ });
 });
 
 gulp.task('copy', function () {
  return gulp.src(paths.statics, {base: './src'})
  .pipe(gulp.dest(paths.dest));
 });
 
+function prepareTemplates() {
+ return gulp.src(paths.templates)
+ .pipe(htmlmin({collapseWhitespace: true}))
+ .pipe(templateCache({module: 'monitorrent'}));
+}
+
+gulp.task('templateCache', function () {
+ return prepareTemplates()
+ .pipe(gulp.dest(paths.dest));
+});
+
 gulp.task('less', function () {
  return gulp.src(paths.styles)
  .pipe(less())
@@ -63,28 +86,34 @@ gulp.task('less', function () {
 
 gulp.task('copy-index', ['copy-index-html', 'copy-login-html']);
 
-function preprocessIndexHtmlTpl(mode) {
+function preprocessIndexHtmlTpl(mode, rev) {
  return gulp.src(['./src/index.html'])
  .pipe(preprocess({
- context: { VERSION: pkg.version, MODE: mode }
+ context: { VERSION: pkg.version, MODE: mode, COMMIT_HASH: rev }
  }))
  .pipe(rename(mode + '.html'))
  .pipe(gulp.dest(paths.dest));
 }
 
-gulp.task('copy-index-html', function () {
- return preprocessIndexHtmlTpl('index');
+gulp.task('copy-index-html', function (cb) {
+ git.long(function (rev) {
+ var stream = preprocessIndexHtmlTpl('index', rev);
+ stream.on('end', cb);
+ });
 });
 
-gulp.task('copy-login-html', function () {
- return preprocessIndexHtmlTpl('login');
+gulp.task('copy-login-html', function (cb) {
+ git.long(function (rev) {
+ var stream = preprocessIndexHtmlTpl('login', rev);
+ stream.on('end', cb);
+ });
 });
 
 gulp.task('watch', ['default'], function () {
- gulp.watch(paths.statics, ['copy']);
- gulp.watch(paths.scripts,  ['concat']);
- gulp.watch(paths.styles, ['less']);
- gulp.watch(paths.index_pages, ['copy-index']);
+ gulp.watch(paths.statics,  ['copy']);
+ gulp.watch(paths.scripts.concat(paths.templates), ['concat']);
+ gulp.watch(paths.styles,  ['less']);
+ gulp.watch(paths.index_pages,  ['copy-index']);
 });
 
 gulp.task('release', ['dist'], function () {
@@ -96,7 +125,7 @@ gulp.task('release', ['dist'], function () {
 gulp.task('dist', ['clean-release', 'copy-python', 'copy-webapp', 'copy-desc']);
 
 gulp.task('copy-python', function () {
- return gulp.src(['./**/*.py', '!./monitorrent/tests/**/*.*', '!./monitorrent/tests_functional/*.*', '!./' + paths.release + '/**/*.py'])
+ return gulp.src(['./**/*.py', '!./tests*/**/*.*', '!./' + paths.release + '/**/*.py'])
  .pipe(gulp.dest(paths.release));
 });
 

monitorrent/__init__.py

@@ -0,0 +1 @@
+__version__ = '1.1.0-rc.2'

monitorrent/engine.py

@@ -1,13 +1,18 @@
 from builtins import str
 from builtins import object
 import sys
-import pytz
 import threading
-import html
+
+from queue import PriorityQueue
+from collections import namedtuple
 from datetime import datetime, timedelta
+
+import pytz
+import html
+
 from sqlalchemy import Column, Integer, ForeignKey, Unicode, Enum, func
 from monitorrent.db import Base, DBSession, row2dict, UTCDateTime
-
+from monitorrent.utils.timers import timer
 
 class Logger(object):
  def started(self, start_time):
@@ -44,17 +49,18 @@ def __init__(self, logger, clients_manager):
  def find_torrent(self, torrent_hash):
  return self.clients_manager.find_torrent(torrent_hash)
 
- def add_torrent(self, filename, torrent, old_hash):
+ def add_torrent(self, filename, torrent, old_hash, topic_settings):
  """
  :type filename: str
  :type old_hash: str | None
  :type torrent: Torrent
+ :type topic_settings: clients.TopicSettings | None
  :rtype: datetime
  """
  existing_torrent = self.find_torrent(torrent.info_hash)
  if existing_torrent:
  self.log.info(u"Torrent <b>%s</b> already added" % filename)
- elif self.clients_manager.add_torrent(torrent.raw_content):
+ elif self.clients_manager.add_torrent(torrent.raw_content, topic_settings):
  old_existing_torrent = self.find_torrent(old_hash) if old_hash else None
  if old_existing_torrent:
  self.log.info(u"Updated <b>%s</b>" % filename)
@@ -250,23 +256,31 @@ def get_current_execute_log_details(self, after=None):
 
 
 class EngineRunner(threading.Thread):
+ RunMessage = namedtuple('RunMessage', ['priority', 'ids'])
+ StopMessage = namedtuple('StopMessage', ['priority'])
+
  def __init__(self, logger, trackers_manager, clients_manager, **kwargs):
  """
  :type logger: Logger
  :type trackers_manager: plugin_managers.TrackersManager
  :type clients_manager: plugin_managers.ClientsManager
  """
  interval_param = kwargs.pop('interval', None)
+ last_execute_param = kwargs.pop('last_execute', None)
+
  super(EngineRunner, self).__init__(**kwargs)
  self.logger = logger
  self.trackers_manager = trackers_manager
  self.clients_manager = clients_manager
- self.waiter = threading.Event()
  self.is_executing = False
  self.is_stoped = False
  self._interval = float(interval_param) if interval_param else 7200
- self._last_execute = None
- self._execute_ids = None
+ self._last_execute = last_execute_param
+ self.message_box = PriorityQueue()
+
+ self.timer_cancel = None
+ self._create_timer()
+
  self.start()
 
  @property
@@ -276,6 +290,7 @@ def interval(self):
  @interval.setter
  def interval(self, value):
  self._interval = value
+ self._create_timer()
 
  @property
  def last_execute(self):
@@ -288,39 +303,66 @@ def last_execute(self, value):
  # noinspection PyBroadException
  def run(self):
  while not self.is_stoped:
- self.waiter.wait(self.interval)
- if self.is_stoped:
+ msg = self._receive()
+
+ if isinstance(msg, EngineRunner.StopMessage):
+ self.is_stoped = True
+ self.timer_cancel()
  return
+
+ ids = msg.ids \
+ if isinstance(msg, EngineRunner.RunMessage) \
+ else None
+
  try:
- self._execute()
+ self._execute(ids=ids)
  except:
  pass
- self.waiter.clear()
 
  def stop(self):
- self.is_stoped = True
- self.waiter.set()
+ self.message_box.put(EngineRunner._stop_message())
 
  def execute(self, ids):
- self._execute_ids = ids
- self.waiter.set()
+ if not self.is_executing:
+ msg = EngineRunner._run_message(ids=ids)
+ self.message_box.put_nowait(msg)
+
+ def _create_timer(self):
+ def timer_fn():
+ msg = EngineRunner._run_message()
+ self.message_box.put_nowait(msg)
+
+ if self.timer_cancel is not None:
+ self.timer_cancel()
+
+ self.timer_cancel = timer(self.interval, timer_fn)
+
+ def _receive(self):
+ return self.message_box.get(block=True)
 
  # noinspection PyBroadException
- def _execute(self):
+ def _execute(self, ids=None):
  caught_exception = None
  self.is_executing = True
  try:
  self.logger.started(datetime.now(pytz.utc))
- self.trackers_manager.execute(Engine(self.logger, self.clients_manager), self._execute_ids)
+ self.trackers_manager.execute(Engine(self.logger, self.clients_manager), ids)
  except:
  caught_exception = sys.exc_info()[0]
  finally:
- self._execute_ids = None
  self.is_executing = False
  self.last_execute = datetime.now(pytz.utc)
  self.logger.finished(self.last_execute, caught_exception)
  return True
 
+ @staticmethod
+ def _run_message(ids=None):
+ return EngineRunner.RunMessage(priority=1, ids=ids)
+
+ @staticmethod
+ def _stop_message():
+ return EngineRunner.StopMessage(priority=0)
+
 
 class DBEngineRunner(EngineRunner):
  DEFAULT_INTERVAL = 7200
@@ -331,10 +373,13 @@ def __init__(self, logger, trackers_manager, clients_manager, **kwargs):
  :type trackers_manager: plugin_managers.TrackersManager
  :type clients_manager: plugin_managers.ClientsManager
  """
- super(DBEngineRunner, self).__init__(logger, trackers_manager, clients_manager, **kwargs)
  execute_settings = self._get_execute_settings()
- self._interval = execute_settings.interval
- self._last_execute = execute_settings.last_execute
+ super(DBEngineRunner, self).__init__(logger,
+ trackers_manager,
+ clients_manager,
+ interval=execute_settings.interval,
+ last_execute=execute_settings.last_execute,
+ **kwargs)
 
  @property
  def interval(self):
@@ -343,6 +388,7 @@ def interval(self):
  @interval.setter
  def interval(self, value):
  self._interval = value
+ self._create_timer()
  self._update_execute_settings()
 
  @property