diff --git a/runbot_janitor/__init__.py b/runbot_janitor/__init__.py deleted file mode 100644 index a6630323..00000000 --- a/runbot_janitor/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- encoding: utf-8 -*- -############################################################################## -# -# Odoo, Open Source Management Solution -# This module copyright (C) 2010 - 2014 Savoir-faire Linux -# (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## -from . import models diff --git a/runbot_janitor/__openerp__.py b/runbot_janitor/__openerp__.py deleted file mode 100644 index 2c05f5cf..00000000 --- a/runbot_janitor/__openerp__.py +++ /dev/null @@ -1,53 +0,0 @@ -# -*- encoding: utf-8 -*- -############################################################################## -# -# Odoo, Open Source Management Solution -# This module copyright (C) 2010 - 2014 Savoir-faire Linux -# (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -{ - 'name': 'Runbot Janitor', - 'category': 'Website', - 'summary': 'Aggressively clean filesystem databases and processes', - 'version': '1.0', - 'description': """ -Runbot Janitor -============== - -Aggressively clean filesystem databases and processes -Start a cron which will look in runbot's build directories -(runbot/static/build) and identify all build directories which have no running -builds (runbot.build). - -Using the names of these directories, cleanup by: - * killing processes which run executables in those directories or are - connected to databases matching the directory names - * drop all databases whose names match the directory names - * delete the directory and its contents - -Contributors ------------- -* Sandy Carter (sandy.carter@savoirfairelinux.com) -* Jordi Riera (jordi.riera@savoirfairelinux.com) -""", - 'author': "Savoir-faire Linux,Odoo Community Association (OCA)", - 'depends': ['runbot'], - 'data': [ - ], - 'installable': True, -} diff --git a/runbot_janitor/models/__init__.py b/runbot_janitor/models/__init__.py deleted file mode 100644 index c2f2e77c..00000000 --- a/runbot_janitor/models/__init__.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- encoding: utf-8 -*- -############################################################################## -# -# Odoo, Open Source Management Solution -# This module copyright (C) 2010 - 2014 Savoir-faire Linux -# (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -from . import runbot_repo diff --git a/runbot_janitor/models/runbot_repo.py b/runbot_janitor/models/runbot_repo.py deleted file mode 100644 index b24f9168..00000000 --- a/runbot_janitor/models/runbot_repo.py +++ /dev/null @@ -1,199 +0,0 @@ -# -*- encoding: utf-8 -*- -############################################################################## -# -# Odoo, Open Source Management Solution -# This module copyright (C) 2010 - 2014 Savoir-faire Linux -# (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## - -import os -import shutil -import re -import pwd -import time -import psutil -import signal -from datetime import datetime, timedelta - -from contextlib import closing - -import logging - -from openerp import models, api, SUPERUSER_ID, tools, sql_db -from openerp.tools import DEFAULT_SERVER_DATETIME_FORMAT as DATETIME_FMT - -_logger = logging.getLogger(__name__) -_logger.setLevel(logging.DEBUG) - - -def exp_list_posix_user(): - """Rewrite/simplified version of openerp.service.exp_list() - Lists all databases owned by the current posix user instead of the db_user - from odoo config. - The reason for this is because runbot creates databases with the posix - user. - :returns list of databases owned by the posix user - """ - chosen_template = tools.config['db_template'] - templates_list = {'template0', 'template1', 'postgres', chosen_template} - db = sql_db.db_connect('postgres') - with closing(db.cursor()) as cr: - db_user = pwd.getpwuid(os.getuid())[0] - cr.execute(""" -SELECT datname -FROM pg_database -WHERE datdba=( - SELECT usesysid - FROM pg_user - WHERE usename=%s -) AND datname NOT IN %s order by datname""", (db_user, tuple(templates_list))) - res = [tools.ustr(name) for (name,) in cr.fetchall()] - res.sort() - return res - -initialized = False - - -class RunbotRepo(models.Model): - """Aggressively clean filesystem databases and processes.""" - _inherit = "runbot.repo" - - def __init__(self, pool, cr): - """On reinitialisation of db, mark all builds untouched since more than 1 day - as done""" - super(RunbotRepo, self).__init__(pool, cr) - global initialized - if initialized: - return - initialized = True - runbot_build = pool['runbot.build'] - yesterday = (datetime.now() - timedelta(1)).strftime(DATETIME_FMT) - domain = [('state', '!=', 'done'), - ('write_date', '<', yesterday), - ] - ids = pool['runbot.build'].search(cr, SUPERUSER_ID, domain) - if ids: - _logger.info('marking %d builds as done', len(ids)) - runbot_build.write(cr, SUPERUSER_ID, ids, {'state': 'done'}) - - @api.model - def cron(self): - """Overcharge cron, add clean up subroutine before the general cron.""" - self.clean_up() - return super(RunbotRepo, self).cron() - - def clean_up(self): - """Examines the build directory, identify leftover builds then - call the cleans: filesystem, database, process - - Leftover builds will have state done - Skip if the build directory hasn't been created yet - """ - self.clean_up_pids() - build_root = os.path.join(self.root(), 'build') - if not os.path.exists(build_root): - return - build_dirs = set( - filter( - # Don't consider old build which have been cleaned except - # and have only logs left - lambda d: os.listdir(os.path.join(build_root, d)) != ['logs'], - os.listdir(build_root) - ) - ) - valid_builds = [b.dest for b in self.env['runbot.build'].search([ - ('dest', 'in', list(build_dirs)), - ('state', '!=', 'done') - ])] - _logger.debug("build_dirs = %s", build_dirs) - _logger.debug("valid_builds = %s", valid_builds) - for pattern in build_dirs.difference(valid_builds): - _logger.info("Runbot Janitor Cleaning up Residue: %s", pattern) - try: - self.clean_up_database(pattern) - except OSError as e: - _logger.error('Error in database cleanup: %s', e) - try: - self.clean_up_process(pattern) - except OSError as e: - _logger.error('Error in process cleanup: %s', e) - try: - self.clean_up_filesystem(pattern) - except OSError as e: - _logger.error('Error in file system cleanup: %s', e) - - def clean_up_pids(self): - """Kill all done pids which are still running - """ - for build in self.env['runbot.build'].search([ - ('pid', 'in', psutil.pids()), - ('state', '=', 'done') - ]): - _logger.debug("Killing pid %s", build.pid) - try: - os.kill(build.pid, signal.SIGKILL) - except OSError, exc: - _logger.warning('Could not kill pid %d: %s', build.pid, exc) - build.pid = False - - def clean_up_database(self, pattern): - """Drop all databases whose names match the directory names matching - the pattern. - - :param pattern:string - """ - runbot_build = self.env['runbot.build'] - regex = re.compile(r'{}.*'.format(pattern)) - db_list = exp_list_posix_user() - time.sleep(1) # Give time for the cursor to close properly - for db_name in filter(regex.match, db_list): - _logger.debug("Dropping %s", db_name) - runbot_build.pg_dropdb(dbname=db_name) - - def clean_up_process(self, pattern): - """Kill processes which run executables in those directories or are - connected to databases matching the directory names matching - the pattern. - - :param pattern: string - """ - regex = re.compile(r'.*-d {}.*'.format(pattern)) - for process in psutil.process_iter(): - if regex.match(" ".join(process.cmdline())): - _logger.debug("Killing pid %s", process.pid) - process.kill() - - def clean_up_filesystem(self, pattern): - """Delete the directory and its contents matching the pattern. - - If there are logs, delete everything except those - - :param pattern: string - """ - pattern_path = os.path.join(self.root(), 'build', pattern) - log_dir = os.path.join(pattern_path, 'logs') - if os.path.isdir(log_dir) and os.listdir(log_dir): - for name in os.listdir(pattern_path): - path = os.path.join(pattern_path, name) - if name == 'logs': - continue - if os.path.isdir(path): - shutil.rmtree(path) - else: - os.unlink(path) - else: - shutil.rmtree(pattern_path) diff --git a/runbot_janitor/tests/__init__.py b/runbot_janitor/tests/__init__.py deleted file mode 100644 index eae0cb06..00000000 --- a/runbot_janitor/tests/__init__.py +++ /dev/null @@ -1,22 +0,0 @@ -# -*- encoding: utf-8 -*- -############################################################################## -# -# Odoo, Open Source Management Solution -# This module copyright (C) 2010 - 2014 Savoir-faire Linux -# (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## -from . import test_runbot_repo diff --git a/runbot_janitor/tests/test_runbot_repo.py b/runbot_janitor/tests/test_runbot_repo.py deleted file mode 100644 index 1ae3b711..00000000 --- a/runbot_janitor/tests/test_runbot_repo.py +++ /dev/null @@ -1,123 +0,0 @@ -# -*- encoding: utf-8 -*- -############################################################################## -# -# Odoo, Open Source Management Solution -# This module copyright (C) 2010 - 2014 Savoir-faire Linux -# (). -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# -############################################################################## -import os -import shutil -import tempfile -import subprocess -import sys -import psutil -import signal - -import logging - -from openerp.tests import TransactionCase -from ..models.runbot_repo import exp_list_posix_user - -_logger = logging.getLogger(__name__) - - -class TestRunbotRepo(TransactionCase): - """Aggressively clean filesystem databases and processes.""" - - def setUp(self): - """ - Create a temp build dir with a python file in it - Run a process from inside build dir - Create a database with similar name to temp build dir - """ - super(TestRunbotRepo, self).setUp() - self.runbot_repo = self.env['runbot.repo'] - self.runbot_build = self.env['runbot.build'] - - # Filesystem - self.root = os.path.join(self.runbot_repo.root(), 'build') - if not os.path.exists(self.root): - os.mkdir(self.root) - self.build_dir = tempfile.mkdtemp(dir=self.root) - self.build_basename = os.path.basename(self.build_dir) - self.build_file_handle = tempfile.mkstemp( - suffix=".py", dir=self.build_dir - ) - - # Database - self.base_database = self.build_basename + "-base" - self.all_database = self.build_basename + "-all" - self.runbot_build.pg_createdb(dbname=self.base_database) - self.runbot_build.pg_createdb(dbname=self.all_database) - - # Process - self.build_filename = self.build_file_handle[1] - with open(self.build_filename, 'w') as f: - f.write("from time import sleep; sleep(60)") - self.process = subprocess.Popen( - [sys.executable, self.build_filename, '-d', self.all_database], - ) - - def tearDown(self): - """Delete temp build dir, kill process and drop database""" - if self.process.returncode is None: - self.process.kill() - if os.path.isdir(self.build_dir): - shutil.rmtree(self.build_dir) - # Database - self.runbot_build.pg_dropdb(dbname=self.build_basename + "-base") - self.runbot_build.pg_dropdb(dbname=self.build_basename + "-all") - - def test_clean_up_process(self): - """Kill processes which run executables in those directories or are - connected to databases matching the directory names matching - the pattern. - - :param pattern: string - """ - self.assertIn(self.process.pid, psutil.pids()) - self.assertIsNone(self.process.returncode) - self.runbot_repo.clean_up_process(self.build_basename) - self.process.wait() - self.assertEqual(self.process.returncode, -signal.SIGKILL) - - def test_clean_up_database(self): - """Drop all databases whose names match the directory names matching - the pattern. - - :param pattern:string - """ - db_list = exp_list_posix_user() - self.assertIn(self.base_database, db_list) - self.assertIn(self.all_database, db_list) - - # cleaning all the test database - self.runbot_repo.clean_up_database(self.build_basename) - - # Need to refresh the db list - db_list = exp_list_posix_user() - self.assertNotIn(self.base_database, db_list) - self.assertNotIn(self.all_database, db_list) - - def test_clean_up_filesystem(self): - """Delete the directory and its contents matching the pattern. - - :param pattern: string - """ - self.assertTrue(os.path.isdir(self.build_dir)) - self.runbot_repo.clean_up_filesystem(self.build_basename) - self.assertFalse(os.path.isdir(self.build_dir))