Release 0.9.2 #711

0.9.2

.travis.yml

@@ -37,6 +37,7 @@ matrix:
       install:
         - sed -i '/WTF_CSRF_ENABLED = True/c\WTF_CSRF_ENABLED = False' `pwd`/env-config/config.py
         - pip install bandit
+        - pip install pylint
 
       script:
         - coverage run -a -m py.test security_monkey/tests/auditors || exit 1
@@ -46,6 +47,7 @@ matrix:
         - coverage run -a -m py.test security_monkey/tests/interface || exit 1
         - coverage run -a -m py.test security_monkey/tests/utilities || exit 1
         - bandit -r -ll -ii -x security_monkey/tests .
+        - pylint -E -d E1101,E0611,F0401 --ignore=service.py,datastore.py,datastore_utils.py,watcher.py security_monkey
 
       after_success:
         - coveralls

Dockerfile

@@ -16,7 +16,7 @@
 FROM ubuntu:14.04
 MAINTAINER Netflix Open Source Development <talent@netflix.com>
 
-ENV SECURITY_MONKEY_VERSION=v0.9.1 \
+ENV SECURITY_MONKEY_VERSION=v0.9.2 \
     SECURITY_MONKEY_SETTINGS=/usr/local/src/security_monkey/env-config/config-docker.py
 
 RUN apt-get update &&\

dart/pubspec.yaml

@@ -1,6 +1,6 @@
 name: security_monkey
 description: An AWS Policy Monitoring and Alerting Tool
-version: 0.9.1
+version: 0.9.2
 dependencies:
   angular: "^1.1.2+2"
   angular_ui: ">=0.6.8 <0.7.0"

docker/nginx/Dockerfile

@@ -15,7 +15,7 @@
 FROM nginx:1.11.4
 MAINTAINER Netflix Open Source Development <talent@netflix.com>
 
-ENV SECURITY_MONKEY_VERSION=v0.9.1
+ENV SECURITY_MONKEY_VERSION=v0.9.2
 RUN apt-get update &&\
   apt-get install -y curl git sudo apt-transport-https &&\
   curl https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - &&\

docs/changelog.md

@@ -1,6 +1,52 @@
 Changelog
 =========
 
+v0.9.2 (2017-05-24)
+----------------------------------------
+
+- PR #695 - @mikegrima - Fixing jinja import bug affecting change emails.
+- PR #692 - @LukeKennedy - Reduce number of API calls in Managed Policy watcher.
+- PR #694 - @supertom - GCP Documentation Updates
+- PR #701 - @supertom - Update GCP ServiceAccount Name to use email instead of DisplayName.
+- PR #702 - @rodriguezsergio - Update KMS Auditor. Don't create issue when Effect is Deny for a wildcard principal.
+- PR #697 - @mcpeak - Pylint fixes and TravisCI pylint enforcement.
+- PR #706 - @monkeysecurity Fix bug where batched watchers did not send change alert emails.
+- PR #708 - @redixin - Fix bug in docker config where `SECURITY_MONKEY_POSTGRES_PORT` would not work if passed as a string.
+- PR #714 - @monkeysecurity - Fix bug where change emails from batched watchers had incorrect color in the JSON diff.
+- PR #713 - @monkeysecurity - Fix path to favicon from flask-security jinja templates.
+- PR #709 - @crruthe - Exempt SSO API from CSRF protection.
+- PR #719 - @monkeysecurity - New simplified watcher format for CloudAux Technologies.
+- PR #726 - @monkeysecurity, @willbengtson - Add new SAMLProvider watcher.
+- PR #730 - @monkeysecurity - Fix bug where ephemerals were not respected for CloudAuxWatcher subclasses.
+- PR #727 - @supertom - Fix bug where duplicate GCP names would violate DB's unique constraint. Names now contain project ID.
+- PR #728 - @supertom - Basic Auditor Tests for GCP.
+- @monkeysecurity - Updated link to Ubuntu's SSL documentation.
+- @monkeysecurity - Bumped version of Cryptography dependency.
+- PEP8 updates.
+
+Important Notes:
+- Additional Permissions Required:
+    - "elasticloadbalancing:describelisteners",
+    - "elasticloadbalancing:describerules",
+    - "elasticloadbalancing:describesslpolicies",
+    - "elasticloadbalancing:describetags",
+    - "elasticloadbalancing:describetargetgroups",
+    - "elasticloadbalancing:describetargetgroupattributes",
+    - "elasticloadbalancing:describetargethealth",
+    - "iam:listsamlproviders",
+- New Watcher: ALB (elbv2)
+- ELB (v1) Watcher re-written with boto3 in CloudAux.  Now respects the config value `SECURITYGROUP_INSTANCE_DETAIL` when determining whether to add the instance id's to the ELB definition.
+
+Contributors:
+- @LukeKennedy
+- @rodriguezsergio
+- @redixin
+- @crruthe
+- @supertom
+- @mcpeak
+- @mikegrima
+- @monkeysecurity
+
 v0.9.1 (2017-04-20)
 ----------------------------------------
 

docs/iam_aws.md

@@ -98,6 +98,13 @@ Paste in this JSON with the name "SecurityMonkeyReadOnly":
                 "elasticloadbalancing:describeloadbalancerattributes",
                 "elasticloadbalancing:describeloadbalancerpolicies",
                 "elasticloadbalancing:describeloadbalancers",
+                "elasticloadbalancing:describelisteners",
+                "elasticloadbalancing:describerules",
+                "elasticloadbalancing:describesslpolicies",
+                "elasticloadbalancing:describetags",
+                "elasticloadbalancing:describetargetgroups",
+                "elasticloadbalancing:describetargetgroupattributes",
+                "elasticloadbalancing:describetargethealth",
                 "es:describeelasticsearchdomainconfig",
                 "es:listdomainnames",
                 "iam:getaccesskeylastused",
@@ -122,6 +129,7 @@ Paste in this JSON with the name "SecurityMonkeyReadOnly":
                 "iam:listpolicies",
                 "iam:listrolepolicies",
                 "iam:listroles",
+                "iam:listsamlproviders",
                 "iam:listservercertificates",
                 "iam:listsigningcertificates",
                 "iam:listuserpolicies",

docs/iam_gcp.md

@@ -30,6 +30,12 @@ To restrict which permissions Security Monkey has to your projects, we'll create
 
 ![Add User to Service Account](images/add_user_to_service_account.png "Add User to Service Account")
 
+Enable IAM API
+---------------
+
+For each GCP project you would like Security Monkey to access, you'll need to enable the IAM API.  Visit the [IAM API page](https://console.cloud.google.com/apis/api/iam.googleapis.com/overview) page in the web console
+ and click 'Enable API' at the top of the screen. When dealing with many projects, you might prefer to do this with the gcloud command.  For details on how to enable services with gcloud, visit the
+  [service-management](https://cloud.google.com/service-management/enable-disable#enabling_services) page.  The IAM service name is 'iam.googleapis.com'.
 
 Next:
 -----

docs/instance_launch_gcp.md

@@ -6,10 +6,11 @@ Create an instance running Ubuntu 14.04 LTS using our 'securitymonkey' service a
 Navigate to the [Create Instance page](https://console.developers.google.com/compute/instancesAdd). Fill in the following fields:
 
 -   **Name**: securitymonkey
--   **Zone**: If using GCP Cloud SQL, select the same zone here.
+-   **Zone**: If using GCP Cloud SQL, select the same zone here. [(Zone List)](https://cloud.google.com/compute/docs/regions-zones/regions-zones#available)
 -   **Machine Type**: 1vCPU, 3.75GB (minimum; also known as n1-standard-1)
 -   **Boot Disk**: Ubuntu 14.04 LTS
 -   **Service Account**: securitymonkey
+-   **Firewall**: Allow HTTPS Traffic
 
 Click the *Create* button to create the instance.
 
@@ -23,9 +24,8 @@ Connecting to your new instance:
 
 We will connect to the new instance over ssh with the gcloud command:
 
-    $ gcloud compute ssh <USERNAME>@<PUBLIC_IP_ADDRESS> --zone us-central
+    $ gcloud compute ssh securitymonkey --zone <ZONE>
 
-Replace the first parameter `<USERNAME>` with the username you authenticated gcloud with. Replace the last parameter `<PUBLIC_IP_ADDRESS>` with the Public IP of your instance.
 
 Next:
 -----

docs/quickstart.md

@@ -110,9 +110,9 @@ If you're using the bleeding edge (develop) branch, you will need to compile the
     /usr/lib/dart/bin/pub build
 
     # Copy the compiled Web UI to the appropriate destination
-    mkdir -p /usr/local/src/security_monkey/security_monkey/static/
-    /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/
-    chgrp -R www-data /usr/local/src/security_monkey
+    sudo mkdir -p /usr/local/src/security_monkey/security_monkey/static/
+    sudo /bin/cp -R /usr/local/src/security_monkey/dart/build/web/* /usr/local/src/security_monkey/security_monkey/static/
+    sudo chgrp -R www-data /usr/local/src/security_monkey
 
 ### Configure the Application
 
@@ -197,7 +197,7 @@ For this quickstart guide, we will use a self-signed SSL certificate. In product
 
 There are some great instructions for generating a certificate on the Ubuntu website:
 
-[Ubuntu - Create a Self Signed SSL Certificate](https://help.ubuntu.com/12.04/serverguide/certificates-and-security.html)
+[Ubuntu - Create a Self Signed SSL Certificate](https://help.ubuntu.com/14.04/serverguide/certificates-and-security.html)
 
 The last commands you need to run from that tutorial are in the "Installing the Certificate" section:
 

env-config/config-docker.py

@@ -70,7 +70,7 @@ def env_to_bool(input):
     }
 }
 
-SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%d/%s' % (
+SQLALCHEMY_DATABASE_URI = 'postgresql://%s:%s@%s:%s/%s' % (
     os.getenv('SECURITY_MONKEY_POSTGRES_USER', 'postgres'),
     os.getenv('SECURITY_MONKEY_POSTGRES_PASSWORD', 'securitymonkeypassword'),
     os.getenv('SECURITY_MONKEY_POSTGRES_HOST', 'localhost'),

scripts/secmonkey_role_setup.py

@@ -87,6 +87,13 @@
            "elasticloadbalancing:describeloadbalancerattributes",
            "elasticloadbalancing:describeloadbalancerpolicies",
            "elasticloadbalancing:describeloadbalancers",
+           "elasticloadbalancing:describelisteners",
+           "elasticloadbalancing:describerules",
+           "elasticloadbalancing:describesslpolicies",
+           "elasticloadbalancing:describetags",
+           "elasticloadbalancing:describetargetgroups",
+           "elasticloadbalancing:describetargetgroupattributes",
+           "elasticloadbalancing:describetargethealth",
            "es:describeelasticsearchdomainconfig",
            "es:listdomainnames",
            "iam:getaccesskeylastused",
@@ -111,6 +118,7 @@
            "iam:listpolicies",
            "iam:listrolepolicies",
            "iam:listroles",
+           "iam:listsamlproviders",
            "iam:listservercertificates",
            "iam:listsigningcertificates",
            "iam:listuserpolicies",

security_monkey/__init__.py

@@ -23,7 +23,7 @@
 import stat
 
 ### VERSION ###
-__version__ = '0.9.1'
+__version__ = '0.9.2'
 
 ### FLASK ###
 from flask import Flask

security_monkey/auditors/elb.py

@@ -25,8 +25,10 @@
 from security_monkey.datastore import NetworkWhitelistEntry
 from security_monkey.datastore import Item
 from security_monkey.watchers.security_group import SecurityGroup
+from collections import defaultdict
 
 import ipaddr
+import json
 
 # From https://docs.aws.amazon.com/ElasticLoadBalancing/latest/DeveloperGuide/elb-security-policy-table.html
 DEPRECATED_CIPHERS = [
@@ -146,14 +148,14 @@ def check_internet_scheme(self, elb_item):
         """
         alert when an ELB has an "internet-facing" scheme.
         """
-        scheme = elb_item.config.get('scheme', None)
-        vpc = elb_item.config.get('vpc_id', None)
+        scheme = elb_item.config.get('Scheme', None)
+        vpc = elb_item.config.get('VPCId', None)
         if scheme and scheme == u"internet-facing" and not vpc:
             self.add_issue(1, 'ELB is Internet accessible.', elb_item)
         elif scheme and scheme == u"internet-facing" and vpc:
             # Grab each attached security group and determine if they contain
             # a public IP
-            security_groups = elb_item.config.get('security_groups', [])
+            security_groups = elb_item.config.get('SecurityGroups', [])
             sg_items = self.get_watcher_support_items(SecurityGroup.index, elb_item.account)
             for sgid in security_groups:
                 for sg in sg_items:
@@ -175,25 +177,35 @@ def check_internet_scheme(self, elb_item):
 
     def check_listener_reference_policy(self, elb_item):
         """
-        alert when an SSL listener is not using the ELBSecurity Policy-2014-10 policy.
+        alert when an SSL listener is not using the latest reference policy.
         """
-        listeners = elb_item.config.get('listeners')
-        for listener in listeners:
-            for policy in listener['policies']:
-                policy_type = policy.get("type", None)
-                if policy_type and policy_type == "SSLNegotiationPolicyType":
-                    reference_policy = policy.get('reference_security_policy', None)
-                    self._process_reference_policy(reference_policy, policy['name'], listener['load_balancer_port'], elb_item)
-                    if not reference_policy:
-                        self._process_custom_listener_policy(policy, listener['load_balancer_port'], elb_item)
+        policy_port_map = defaultdict(list)
+        for listener in elb_item.config.get('ListenerDescriptions'):
+            if len(listener.get('PolicyNames', [])) > 0:
+                for name in listener.get('PolicyNames', []):
+                    policy_port_map[name].append(listener['LoadBalancerPort'])
+
+        policies = elb_item.config.get('PolicyDescriptions')
+        for policy_name, policy in policies.items():
+            policy_type = policy.get("type", None)
+            if policy_type and policy_type == "SSLNegotiationPolicyType":
+                reference_policy = policy.get('reference_security_policy', None)
+                self._process_reference_policy(reference_policy, policy_name, json.dumps(policy_port_map[policy_name]), elb_item)
+                if not reference_policy:
+                    self._process_custom_listener_policy(policy_name, policy, json.dumps(policy_port_map[policy_name]), elb_item)
 
     def check_logging(self, elb_item):
         """
         Alert when elb logging is not enabled
         """
-        logging = elb_item.config.get('is_logging')
+        logging = elb_item.config.get('Attributes', {}).get('AccessLog', {})
         if not logging:
             self.add_issue(1, 'ELB is not configured for logging.', elb_item)
+            return
+
+        if not logging.get('Enabled'):
+            self.add_issue(1, 'ELB is not configured for logging.', elb_item)
+            return
 
     def _process_reference_policy(self, reference_policy, policy_name, port, elb_item):
         notes = "Policy {0} on port {1}".format(policy_name, port)
@@ -266,20 +278,20 @@ def _process_reference_policy(self, reference_policy, policy_name, port, elb_ite
         notes = reference_policy
         self.add_issue(10, "Unknown reference policy.", elb_item, notes=notes)
 
-    def _process_custom_listener_policy(self, policy, port, elb_item):
+    def _process_custom_listener_policy(self, policy_name, policy, port, elb_item):
         """
         Alerts on:
             sslv2
             sslv3
             missing server order preference
             deprecated ciphers
         """
-        notes = "Policy {0} on port {1}".format(policy['name'], port)
+        notes = "Policy {0} on port {1}".format(policy_name, port)
 
-        if policy.get('sslv2', None):
+        if policy.get('protocols', {}).get('sslv2', None):
             self.add_issue(10, "SSLv2 is enabled", elb_item, notes=notes)
 
-        if policy.get('sslv3', None):
+        if policy.get('protocols', {}).get('sslv3', None):
             self.add_issue(10, "SSLv3 is enabled", elb_item, notes=notes)
 
         server_defined_cipher_order = policy.get('server_defined_cipher_order', None)

security_monkey/auditors/kms.py

@@ -85,7 +85,7 @@ def check_for_kms_policy_with_foreign_account(self, kms_item):
 
                     principal_account_ids = set()
                     for arn in aws_principal:
-                        if arn == "*" and not condition_accounts:
+                        if arn == "*" and not condition_accounts and "allow" == statement.get('Effect').lower():
                             notes = "An KMS policy where { 'Principal': { 'AWS': '*' } } must also have"
                             notes += " a {'Condition': {'StringEquals': { 'kms:CallerAccount': '<AccountNumber>' } } }"
                             notes += " or it is open to the world."

security_monkey/cloudaux_batched_watcher.py

@@ -0,0 +1,72 @@
+from security_monkey.cloudaux_watcher import CloudAuxWatcher
+from security_monkey.cloudaux_watcher import CloudAuxChangeItem
+from security_monkey.decorators import record_exception
+from cloudaux.decorators import iter_account_region
+
+
+class CloudAuxBatchedWatcher(CloudAuxWatcher):
+
+    def __init__(self, **kwargs):
+        super(CloudAuxBatchedWatcher, self).__init__(**kwargs)
+        self.batched_size = 100
+        self.done_slurping = False
+
+    def slurp_list(self):
+        self.prep_for_batch_slurp()
+
+        @record_exception(source='{index}-watcher'.format(index=self.index), pop_exception_fields=True)
+        def invoke_list_method(**kwargs):
+            return self.list_method(**kwargs['conn_dict'])
+
+        @iter_account_region(self.service_name, accounts=self.account_identifiers,
+            regions=self._get_regions(), conn_type='dict')
+        def get_item_list(**kwargs):
+            kwargs, exception_map = self._add_exception_fields_to_kwargs(**kwargs)
+            items = invoke_list_method(**kwargs)
+
+            if not items:
+                self.done_slurping = True
+                items = list()
+
+            return items, exception_map
+
+        items, exception_map = self._flatten_iter_response(get_item_list())
+        self.total_list.extend(items)
+
+        return items, exception_map
+
+    def slurp(self):
+
+        @record_exception(source='{index}-watcher'.format(index=self.index), pop_exception_fields=True)
+        def invoke_get_method(item, **kwargs):
+            return self.get_method(item, **kwargs['conn_dict'])
+
+        @iter_account_region(self.service_name, accounts=self.account_identifiers,
+            regions=self._get_regions(), conn_type='dict')
+        def slurp_items(**kwargs):
+            item_list = list()
+            kwargs, exception_map = self._add_exception_fields_to_kwargs(**kwargs)
+            item_counter = self.batch_counter * self.batched_size
+            while self.batched_size - len(item_list) > 0 and not self.done_slurping:
+                cursor = self.total_list[item_counter]
+                item_name = self.get_name_from_list_output(cursor)
+                if item_name and self.check_ignore_list(item_name):
+                    item_counter += 1
+                    if item_counter == len(self.total_list):
+                        self.done_slurping = True
+                    continue
+
+                item_details = invoke_get_method(cursor, name=item_name, **kwargs)
+                if item_details:
+                    item = CloudAuxChangeItem.from_item(
+                        name=item_name,
+                        item=item_details,
+                        override_region=self.override_region, **kwargs)
+                    item_list.append(item)
+                item_counter += 1
+                if item_counter == len(self.total_list):
+                    self.done_slurping = True
+            self.batch_counter += 1
+            return item_list, exception_map
+
+        return self._flatten_iter_response(slurp_items())