diff --git a/.gitignore b/.gitignore index b1fd7e2..9ccd7b2 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,7 @@ celerybeat-schedule # Spyder project settings .spyderproject + +# Vagrant +Vagrantfile +.vagrant/ diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000..ccb232f --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,114 @@ +.. highlight:: shell + +============ +Contributing +============ + +Contributions are welcome, and they are greatly appreciated! Every +little bit helps, and credit will always be given. + +You can contribute in many ways: + +Types of Contributions +---------------------- + +Report Bugs +~~~~~~~~~~~ + +Report bugs at https://github.com/bonclay7/aws-amicleaner/issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +Fix Bugs +~~~~~~~~ + +Look through the GitHub issues for bugs. Anything tagged with "bug" +and "help wanted" is open to whoever wants to implement it. + +Implement Features +~~~~~~~~~~~~~~~~~~ + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +Write Documentation +~~~~~~~~~~~~~~~~~~~ + +amicleaner could always use more documentation, whether as part of the +official amicleaner docs, in docstrings, or even on the web in blog posts, +articles, and such. + +Submit Feedback +~~~~~~~~~~~~~~~ + +The best way to send feedback is to file an issue at https://github.com/bonclay7/aws-amicleaner/issues. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +Get Started! +------------ + +Ready to contribute? Here's how to set up `aws-amicleaner` for local development. + +1. Fork the `aws-amicleaner` repo on GitHub. +2. Clone your fork locally:: + + $ git clone git@github.com:bonclay7/aws-amicleaner.git + +3. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:: + + $ mkvirtualenv amicleaner + $ cd aws-amicleaner/ + $ python setup.py develop + +4. Create a branch for local development:: + + $ git checkout -b name-of-your-bugfix-or-feature + + Now you can make your changes locally. + +5. When you're done making changes, check that your changes pass flake8 and the tests, including testing other Python versions with tox:: + + $ flake8 amicleaner tests + $ python setup.py test or py.test + $ tox + + To get flake8 and tox, just pip install them into your virtualenv. + +6. Commit your changes and push your branch to GitHub:: + + $ git add . + $ git commit -m "Your detailed description of your changes." + $ git push origin name-of-your-bugfix-or-feature + +7. Submit a pull request through the GitHub website. + +Pull Request Guidelines +----------------------- + +Before you submit a pull request, check that it meets these guidelines: + +1. The pull request should include tests. +2. If the pull request adds functionality, the docs should be updated. Put + your new functionality into a function with a docstring, and add the + feature to the list in README.rst. +3. The pull request should work for Python 2.6, 2.7 and for PyPy. Check + https://travis-ci.org/bonclay7/aws-amicleaner/pull_requests + and make sure that the tests pass for all supported Python versions. + +Tips +---- + +To run a subset of tests:: + +$ py.test tests.test_amicleaner + diff --git a/HISTORY.rst b/HISTORY.rst new file mode 100644 index 0000000..c237a15 --- /dev/null +++ b/HISTORY.rst @@ -0,0 +1,8 @@ +======= +History +======= + +0.1.0 (2016-08-22) +------------------ + +* First release on PyPI. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b30e242 --- /dev/null +++ b/LICENSE @@ -0,0 +1,11 @@ + +MIT License + +Copyright (c) 2016, Guy Rodrigue Koffi + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/README.md b/README.md deleted file mode 100644 index f695bda..0000000 --- a/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# aws-amicleaner -Cleanup your old unused ami and related snapshots - -[![Circle CI](https://circleci.com/gh/bonclay7/aws-amicleaner/tree/master.svg?style=svg)](https://circleci.com/gh/bonclay7/aws-amicleaner/tree/master) -[![codecov.io](https://codecov.io/github/bonclay7/aws-amicleaner/coverage.svg?branch=master)](https://codecov.io/github/bonclay7/aws-amicleaner?branch=master) - -## Description - -This tools permits to clean your custom [Amazon Machine Images (AMI)] (http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) and related [EBS Snapshots] (http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSSnapshots.html). - -You can either run in `fetch and clean` mode where the tool will retrieve all your private __AMIs__ and EC2 instances, exclude AMIs being holded by your EC2 instances (it can be useful if you use autoscaling, and so on ...). It applies a filter based on their __names__ or __tags__ and a number of __previous AMIs__ you want to keep. - -It can simply remove AMIs with a list of provided ids ! - -## Prerequisites - -- [awscli](http://docs.aws.amazon.com/cli/latest/userguide/installing.html) -- [python 2.7](https://www.python.org/downloads/release/python-2710/) -- [pyhton pip](https://pip.pypa.io/en/stable/installing/) - -This tool assumes your AWS credentials sourced, either with aws credentials variables : - -```bash -export AWS_DEFAULT_REGION='your region' -export AWS_ACCESS_KEY_ID='with token Access ID' -export AWS_SECRET_ACCESS_KEY='with token AWS Secret' -``` - -or with `awscli` : - -```bash -export AWS_PROFILE=profile-name -``` - -## How does it work ? - -To run the script properly, your `aws` user must have at least -these permissions in `iam`: - -```json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Stmt1458638250000", - "Effect": "Allow", - "Action": [ - "ec2:DeleteSnapshot", - "ec2:DeregisterImage", - "ec2:DescribeImages", - "ec2:DescribeInstances", - "ec2:DescribeSnapshots" - ], - "Resource": [ - "arn:aws:ec2:::*" - ] - } - ] -} -``` - -### Getting help - -```bash -amicleaner/cli.py --help -``` - -### Clean a list of AMIs - -```bash -amicleaner/cli.py --from-ids ami-abcdef01 ami-abcdef02 -``` - -### Fetch and clean - -Print report of groups and amis to be cleaned -```bash -amicleaner/cli.py --full-report -``` - -Keep previous number of AMIs -```bash -amicleaner/cli.py --full-report --keep-previous 10 -``` - -Regroup by name or tags -```bash -amicleaner/cli.py --mapping-key tags --mapping-values role env -``` - -Skip confirmation, can be useful for automation -```bash -amicleaner/cli.py -f --keep-previous 2 -``` - -### Using virtual env - -```bash -$ virtualenv env -$ . env/bin/activate - (env) aws-amicleaner $ pip install -r requirements.txt - (env) aws-amicleaner $ amicleaner/cli.py -``` - -## Contributing - -Issues reporting and pull requests are welcome ! - -## License - -``` -The MIT License (MIT) - -Copyright (c) 2016 Guy Rodrigue Koffi - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. -``` diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..d125b62 --- /dev/null +++ b/README.rst @@ -0,0 +1,132 @@ +aws-amicleaner +============== + +Cleanup your old unused ami and related snapshots + +|Circle CI| |codecov.io| |pypi| + +Description +----------- + +This tools permits to clean your custom `Amazon Machine Images (AMI) +`__ and +related `EBS Snapshots +`__. + +You can either run in ``fetch and clean`` mode where the tool will +retrieve all your private **AMIs** and EC2 instances, exclude AMIs being +holded by your EC2 instances (it can be useful if you use autoscaling, +and so on ...). It applies a filter based on their **names** or **tags** +and a number of **previous AMIs** you want to keep. + +It can simply remove AMIs with a list of provided ids ! + +Prerequisites +------------- + +- `awscli `__ +- `python + 2.7 `__ +- `pyhton pip `__ + +This tool assumes your AWS credentials sourced, either with aws +credentials variables : + +.. code:: bash + + export AWS_DEFAULT_REGION='your region' + export AWS_ACCESS_KEY_ID='with token Access ID' + export AWS_SECRET_ACCESS_KEY='with token AWS Secret' + +or with ``awscli`` : + +.. code:: bash + + export AWS_PROFILE=profile-name + +How does it work ? +------------------ + +To run the script properly, your ``aws`` user must have at least these +permissions in ``iam``: + +.. code:: json + + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Stmt1458638250000", + "Effect": "Allow", + "Action": [ + "ec2:DeleteSnapshot", + "ec2:DeregisterImage", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeSnapshots" + ], + "Resource": [ + "arn:aws:ec2:::*" + ] + } + ] + } + +Getting help +~~~~~~~~~~~~ + +.. code:: bash + + amicleaner/cli.py --help + +Clean a list of AMIs +~~~~~~~~~~~~~~~~~~~~ + +.. code:: bash + + amicleaner/cli.py --from-ids ami-abcdef01 ami-abcdef02 + +Fetch and clean +~~~~~~~~~~~~~~~ + +Print report of groups and amis to be cleaned + +.. code:: bash + + amicleaner/cli.py --full-report + +Keep previous number of AMIs + +.. code:: bash + + amicleaner/cli.py --full-report --keep-previous 10 + +Regroup by name or tags + +.. code:: bash + + amicleaner/cli.py --mapping-key tags --mapping-values role env + +Skip confirmation, can be useful for automation + +.. code:: bash + + amicleaner/cli.py -f --keep-previous 2 + +Using virtual env +~~~~~~~~~~~~~~~~~ + +.. code:: bash + + $ virtualenv env + $ . env/bin/activate + (env) aws-amicleaner $ pip install -r requirements.txt + (env) aws-amicleaner $ amicleaner/cli.py + + +.. |Circle CI| image:: https://circleci.com/gh/bonclay7/aws-amicleaner/tree/master.svg?style=svg + :target: https://circleci.com/gh/bonclay7/aws-amicleaner/tree/master +.. |codecov.io| image:: https://codecov.io/github/bonclay7/aws-amicleaner/coverage.svg?branch=master + :target: https://codecov.io/github/bonclay7/aws-amicleaner?branch=master +.. |pypi| image:: https://img.shields.io/pypi/v/aws-amicleaner.svg + :target: https://pypi.python.org/pypi/aws-amicleaner diff --git a/amicleaner/__init__.py b/amicleaner/__init__.py index e69de29..5a59d05 100644 --- a/amicleaner/__init__.py +++ b/amicleaner/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- + +__title__ = 'amicleaner' +__version__ = '0.1.0' +__short_version__ = '.'.join(__version__.split('.')[:2]) +__author__ = 'Guy Rodrigue Koffi' +__author_email__ = 'koffirodrigue@gmail.com' +__license__ = 'MIT' diff --git a/amicleaner/cli.py b/amicleaner/cli.py index 1d163ed..92ea663 100755 --- a/amicleaner/cli.py +++ b/amicleaner/cli.py @@ -1,6 +1,9 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- + import sys +from amicleaner import __version__ from core import AMICleaner, OrphanSnapshotCleaner from resources.config import MAPPING_KEY, MAPPING_VALUES from resources.config import TERM @@ -11,6 +14,7 @@ class App: def __init__(self, args): + self.version = args.version self.mapping_key = args.mapping_key or MAPPING_KEY self.mapping_values = args.mapping_values or MAPPING_VALUES self.keep_previous = args.keep_previous @@ -103,6 +107,10 @@ def print_defaults(self): print TERM.green("mapping_values : {0}".format(self.mapping_values)) print TERM.green("keep_previous : {0}".format(self.keep_previous)) + @staticmethod + def print_version(): + print(__version__) + def run_cli(self): if self.check_orphans: @@ -141,7 +149,11 @@ def main(): sys.exit(1) app = App(args) - app.run_cli() + + if app.version is True: + app.print_version() + else: + app.run_cli() if __name__ == "__main__": diff --git a/amicleaner/core.py b/amicleaner/core.py index d8aa3e1..629158f 100644 --- a/amicleaner/core.py +++ b/amicleaner/core.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- import boto3 from botocore.exceptions import ClientError @@ -9,8 +10,8 @@ class OrphanSnapshotCleaner: """ Finds and removes ebs snapshots left orphaned """ - def __init__(self): - self.ec2 = boto3.client('ec2') + def __init__(self, ec2=None): + self.ec2 = ec2 or boto3.client('ec2') def get_snapshots_filter(self): @@ -25,6 +26,17 @@ def get_snapshots_filter(self): ] }] + def get_owner_id(self, images_json): + + """ Return AWS owner id from a ami json list """ + + images = images_json or [] + + if not images: + return None + + return images[0].get("OwnerId", "") + def fetch(self): """ retrieve orphan snapshots """ @@ -37,7 +49,10 @@ def fetch(self): for ebs in image.get("BlockDeviceMappings") ] snap_filter = self.get_snapshots_filter() - owner_id = resp.get("Images")[0].get("OwnerId") + owner_id = self.get_owner_id(resp.get("Images")) + + if not owner_id: + return [] # all snapshots created for AMIs resp = self.ec2.describe_snapshots( diff --git a/amicleaner/resources/config.py b/amicleaner/resources/config.py index 8e61fc6..fe6a430 100644 --- a/amicleaner/resources/config.py +++ b/amicleaner/resources/config.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- + # set your aws env vars to production from blessings import Terminal diff --git a/amicleaner/resources/models.py b/amicleaner/resources/models.py index 6536696..169a975 100644 --- a/amicleaner/resources/models.py +++ b/amicleaner/resources/models.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- class AMI: diff --git a/amicleaner/utils.py b/amicleaner/utils.py index 28d6fd7..829a04e 100644 --- a/amicleaner/utils.py +++ b/amicleaner/utils.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +# -*- coding: utf-8 -*- + import argparse from prettytable import PrettyTable @@ -61,6 +63,11 @@ def parse_args(args): 'AWS account. Your AWS ' 'credentials must be sourced') + parser.add_argument("-v", "--version", + dest='version', + action="store_true", + help="Prints version and exits") + parser.add_argument("--from-ids", dest='from_ids', nargs='+', diff --git a/circle.yml b/circle.yml index cdba98a..2cdeaf8 100644 --- a/circle.yml +++ b/circle.yml @@ -13,6 +13,6 @@ dependencies: - sudo pip install -r requirements_build.txt test: override: - - py.test -v --ignore=venv --cov . --pep8 . + - py.test -v --ignore=venv --cov . --pep8 tests post: - codecov diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..9518641 --- /dev/null +++ b/setup.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python + +from setuptools import setup, find_packages +from amicleaner import __author__, __author_email__ +from amicleaner import __license__, __version__ + + +with open('README.rst') as readme_file: + readme = readme_file.read() + + +install_requirements = ['awscli', 'argparse', 'boto', + 'boto3', 'prettytable', 'blessings'] + +test_requirements = ['moto', 'pytest', 'pytest-pep8', 'pytest-cov'] + + +setup( + name='aws-amicleaner', + version=__version__, + description='Cleanup tool for AWS AMIs and snapshots', + long_description=readme, + author=__author__, + author_email=__author_email__, + url='https://github.com/bonclay7/aws-amicleaner/', + license=__license__, + packages=find_packages(exclude=['tests']), + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Intended Audience :: Developers', + 'Intended Audience :: System Administrators', + 'Intended Audience :: Information Technology', + 'License :: OSI Approved :: MIT License', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 2.7', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + entry_points={ + 'console_scripts': [ + 'amicleaner = amicleaner.cli:main', + ], + }, + tests_require=test_requirements, + install_requires=install_requirements, +) diff --git a/tests/test_cli.py b/tests/test_cli.py index a301549..3c07ff7 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import json import boto3 @@ -102,3 +104,7 @@ def test_print_failed_snapshots(): def test_print_orphan_snapshots(): assert Printer.print_orphan_snapshots({}) is None assert Printer.print_orphan_snapshots(["ami-one", "ami-two"]) is None + + +def test_print_defaults(): + assert App(parse_args([])).print_defaults() is None diff --git a/tests/test_core.py b/tests/test_core.py index 809d4b4..066189b 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -1,8 +1,10 @@ +# -*- coding: utf-8 -*- + import boto3 from datetime import datetime from moto import mock_ec2 -from amicleaner.core import AMICleaner +from amicleaner.core import AMICleaner, OrphanSnapshotCleaner from amicleaner.resources.models import AMI, AWSEC2Instance, AWSTag @@ -254,3 +256,41 @@ def test_fetch_instances(): # Test fetch instances method assert len(AMICleaner(conn).fetch_instances()) == 1 + + +@mock_ec2 +def test_fetch_snapshots_from_none(): + + cleaner = OrphanSnapshotCleaner() + + assert len(cleaner.get_snapshots_filter()) > 0 + assert type(cleaner.fetch()) is list + assert len(cleaner.fetch()) == 0 + +""" +@mock_ec2 +def test_fetch_snapshots(): + base_ami = "ami-1234abcd" + + conn = boto3.client('ec2') + reservation = conn.run_instances( + ImageId=base_ami, MinCount=1, MaxCount=1 + ) + instance = reservation["Instances"][0] + + # create amis + images = [] + for i in xrange(5): + image = conn.create_image( + InstanceId=instance.get("InstanceId"), + Name="test-ami" + ) + images.append(image.get("ImageId")) + + # deleting two amis, creating orphan snpashots condition + conn.deregister_image(ImageId=images[0]) + conn.deregister_image(ImageId=images[1]) + + cleaner = OrphanSnapshotCleaner() + assert len(cleaner.fetch()) == 0 +""" diff --git a/tests/test_parsing.py b/tests/test_parsing.py index 18de073..2c4623e 100644 --- a/tests/test_parsing.py +++ b/tests/test_parsing.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- + import json from amicleaner.resources.models import AMI, AWSBlockDevice, AWSEC2Instance