From 0462e83ca2164b1cde6a87681ec8ea8a9ea4755c Mon Sep 17 00:00:00 2001 From: "Kazmier, Peter" Date: Wed, 3 Jul 2024 10:40:16 -0400 Subject: [PATCH] chore: release v3.1.0 - Add `URLAccountLoader` plug-in which supports HTTP authentication (basic, digest, oauth2). This plug-in should be preferred over the existing `JSONAccountLoader`, `YAMLAccountLoader`, and `CSVAccountLoader` plug-ins, which will eventually be deprecated. - Add HTTP POST support to the SAML credential provider plug-in. Thanks to @RobertShan2000 for the contribution. --- README.md | 10 + docs/acctload.html | 473 +++++++++++++ docs/cloudwatch.html | 1 + docs/commands/aws/last.html | 166 ++++- docs/commands/aws/list_lambdas.html | 4 +- docs/config.html | 28 +- docs/index.html | 2 +- docs/plugins/accts/index.html | 1004 ++++++++++++++++++++++++++- docs/plugins/creds/aws.html | 36 + docs/plugmgr.html | 1 + docs/session/aws.html | 26 +- src/awsrun/__init__.py | 2 +- 12 files changed, 1665 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index 6302c8e..e0bed96 100644 --- a/README.md +++ b/README.md @@ -313,6 +313,16 @@ includes the following: ## Change Log +### v3.1.0 + +- Add `URLAccountLoader` plug-in which supports HTTP authentication (basic, + digest, oauth2). This plug-in should be preferred over the existing + `JSONAccountLoader`, `YAMLAccountLoader`, and `CSVAccountLoader` plug-ins, + which will eventually be deprecated. + +- Add HTTP POST support to the SAML credential provider plug-in. Thanks to + @RobertShan2000 for the contribution. + ### v3.0.0 - **BREAKING CHANGE**: Installation via `pip install awsurn` no longer installs diff --git a/docs/acctload.html b/docs/acctload.html index 262711d..af5e328 100644 --- a/docs/acctload.html +++ b/docs/acctload.html @@ -176,6 +176,7 @@

Overview

import requests import yaml +from requests.auth import AuthBase from requests_file import FileAdapter from awsrun.cache import PersistentExpiringValue @@ -1032,6 +1033,174 @@

Overview

super().__init__(accts) +class CSVParser: + """Returns a list of dicts from a buffer of CSV text. + + To override options passed to `csv.DictReader`, specify them as keyword + arguments in the constructor. By default, the `delimiter` is `","` and + `skipinitialspace` is `True`. + """ + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.kwargs.setdefault("delimiter", ",") + self.kwargs.setdefault("skipinitialspace", True) + + def __call__(self, text): + buf = io.StringIO(text.strip()) + return list(csv.DictReader(buf, **self.kwargs)) + + +class JSONParser: + """Returns a list or dict from a buffer of JSON-formatted text. + + To override options passed to `json.loads`, specify them as keyword + arguments in the constructor. + """ + + def __init__(self, **kwargs): + self.kwargs = kwargs + + def __call__(self, text): + return json.loads(text, **self.kwargs) + + +class YAMLParser: + """Returns a list or dict from a buffer of YAML-formatted text. + + To override options passed to `yaml.safe_load`, specify them as keyword + arguments in the constructor. + """ + + def __init__(self, **kwargs): + self.kwargs = kwargs + + def __call__(self, text): + return yaml.safe_load(text, **self.kwargs) + + +class HTTPOAuth2(AuthBase): + """Attaches an OAuth2 bearer token to the given `requests.Request` object. + + Use `token_url` to specify the token provider's URL. The `client_id` and + `client_secret` specify the credentials used to authenticate with the + token provider. Three additional keyword parameters are accepted: + + `scope` + : Default is "AppIdClaimsTrust". + + `grant_type` + : Default is "client_credentials". + + `intent` + : Default is "awsrun account loader plugin" + """ + + def __init__( + self, + token_url, + username, + password, + scope="AppIdClaimsTrust", + grant_type="client_credentials", + intent="awsrun account loader plugin", + ): + self.url = token_url + self.data = {} + self.data["client_id"] = username + self.data["client_secret"] = password + self.data["scope"] = scope + self.data["grant_type"] = grant_type + self.data["intent"] = intent + + def _get_token(self): + resp = requests.post(self.url, data=self.data) + resp.raise_for_status() + return resp.json()["access_token"] + + def __call__(self, req): + req.headers["Authorization"] = f"Bearer {self._get_token()}" + return req + + +class URLAccountLoader(MetaAccountLoader): + """Returns an `AccountLoader` with accounts loaded from a URL. + + Loaded accounts will include metadata associated with each account in the + document retrieved from the `url`. File based URLs can be used to load + data from a local file. This data will be parsed as JSON by default. To + override, use `parser` to specify a callable that accepts the text and + returns a list or dict of accounts (see `MetaAccountLoader`). To cache the + results, specify a non-zere number of seconds in `max_age`. The default + location on disk is the system temp directory in a file called + `awsrun.dat`, which can be overrided via `cache_path`. + + Given the following JSON: + + { + "Accounts": [ + {"id": "100200300400", "env": "prod", "status": "active"}, + {"id": "200300400100", "env": "non-prod", "status": "active"}, + {"id": "300400100200", "env": "non-prod", "status": "suspended"} + ] + } + + The account loader will build account objects with the following attribute + names: `id`, `env`, `status`. Assume the above JSON is returned from + http://example.com/accts.json: + + loader = URLAccountLoader('http://example.com/accts.json', path=['Accounts']) + accts = loader.accounts() + + # Let's inspect the 1st account object and its metadata + assert accts[0].id == '100200300400' + assert accts[0].env == 'prod' + assert accts[0].status == 'active' + + URLAccountLoader is a subclass of the `MetaAccountLoader`, which loads + accounts from a set of dicts. As such, the remainder of the parameters in + the constructor -- `id_attr`, `path`, `str_template`, `include_attrs`, and + `exclude_attrs` -- are defined in the constructor of `MetaAccountLoader`. + """ + + def __init__( + self, + url, + parser=JSONParser(), + auth=None, + max_age=0, + id_attr="id", + path=None, + str_template=None, + include_attrs=None, + exclude_attrs=None, + no_verify=False, + cache_path=None, + ): + + session = requests.Session() + session.mount("file://", FileAdapter()) + + def load_cache(): + r = session.get(url, auth=auth, verify=not no_verify) + r.raise_for_status() + return parser(r.text) + + if not cache_path: + cache_path = Path(tempfile.gettempdir(), "awsrun.dat") + + accts = PersistentExpiringValue(load_cache, cache_path, max_age=max_age) + + super().__init__( + accts.value(), + id_attr=id_attr, + path=[] if path is None else path, + str_template=str_template, + include_attrs=[] if include_attrs is None else include_attrs, + exclude_attrs=[] if exclude_attrs is None else exclude_attrs, + ) + + class AbstractAccount: """Abstract base class used by `MetaAccountLoader` to represent an account and its metadata. @@ -2038,6 +2207,7 @@

Subclasses

  • AzureCLIAccountLoader
    •
  • CSVAccountLoader
    •
  • JSONAccountLoader
    •  +
  • URLAccountLoader
    •
  • YAMLAccountLoader
    •

    Methods

    @@ -2737,6 +2907,294 @@

    Inherited members

    +
    +class CSVParser +(**kwargs) +
    +
    +

    Returns a list of dicts from a buffer of CSV text.

    +

    To override options passed to csv.DictReader, specify them as keyword +arguments in the constructor. By default, the delimiter is "," and +skipinitialspace is True.

    +
    + +Expand source code + +
    class CSVParser:
+    """Returns a list of dicts from a buffer of CSV text.
+
+    To override options passed to `csv.DictReader`, specify them as keyword
+    arguments in the constructor. By default, the `delimiter` is `","` and
+    `skipinitialspace` is `True`.
+    """
+
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+        self.kwargs.setdefault("delimiter", ",")
+        self.kwargs.setdefault("skipinitialspace", True)
+
+    def __call__(self, text):
+        buf = io.StringIO(text.strip())
+        return list(csv.DictReader(buf, **self.kwargs))
    +
    +
    +
    +class JSONParser +(**kwargs) +
    +
    +

    Returns a list or dict from a buffer of JSON-formatted text.

    +

    To override options passed to json.loads, specify them as keyword +arguments in the constructor.

    +
    + +Expand source code + +
    class JSONParser:
+    """Returns a list or dict from a buffer of JSON-formatted text.
+
+    To override options passed to `json.loads`, specify them as keyword
+    arguments in the constructor.
+    """
+
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+
+    def __call__(self, text):
+        return json.loads(text, **self.kwargs)
    +
    +
    +
    +class YAMLParser +(**kwargs) +
    +
    +

    Returns a list or dict from a buffer of YAML-formatted text.

    +

    To override options passed to yaml.safe_load, specify them as keyword +arguments in the constructor.

    +
    + +Expand source code + +
    class YAMLParser:
+    """Returns a list or dict from a buffer of YAML-formatted text.
+
+    To override options passed to `yaml.safe_load`, specify them as keyword
+    arguments in the constructor.
+    """
+
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+
+    def __call__(self, text):
+        return yaml.safe_load(text, **self.kwargs)
    +
    +
    +
    +class HTTPOAuth2 +(token_url, username, password, scope='AppIdClaimsTrust', grant_type='client_credentials', intent='awsrun account loader plugin') +
    +
    +

    Attaches an OAuth2 bearer token to the given requests.Request object.

    +

    Use token_url to specify the token provider's URL. The client_id and +client_secret specify the credentials used to authenticate with the +token provider. Three additional keyword parameters are accepted:

    +
    +
    scope
    +
    Default is "AppIdClaimsTrust".
    +
    grant_type
    +
    Default is "client_credentials".
    +
    intent
    +
    Default is "awsrun account loader plugin"
    +
    +
    + +Expand source code + +
    class HTTPOAuth2(AuthBase):
+    """Attaches an OAuth2 bearer token to the given `requests.Request` object.
+
+    Use `token_url` to specify the token provider's URL. The `client_id` and
+    `client_secret` specify the credentials used to authenticate with the
+    token provider. Three additional keyword parameters are accepted:
+
+    `scope`
+    : Default is "AppIdClaimsTrust".
+
+    `grant_type`
+    : Default is "client_credentials".
+
+    `intent`
+    : Default is "awsrun account loader plugin"
+    """
+
+    def __init__(
+        self,
+        token_url,
+        username,
+        password,
+        scope="AppIdClaimsTrust",
+        grant_type="client_credentials",
+        intent="awsrun account loader plugin",
+    ):
+        self.url = token_url
+        self.data = {}
+        self.data["client_id"] = username
+        self.data["client_secret"] = password
+        self.data["scope"] = scope
+        self.data["grant_type"] = grant_type
+        self.data["intent"] = intent
+
+    def _get_token(self):
+        resp = requests.post(self.url, data=self.data)
+        resp.raise_for_status()
+        return resp.json()["access_token"]
+
+    def __call__(self, req):
+        req.headers["Authorization"] = f"Bearer {self._get_token()}"
+        return req
    +
    +

    Ancestors

    + +
    +
    +class URLAccountLoader +(url, parser=<awsrun.acctload.JSONParser object>, auth=None, max_age=0, id_attr='id', path=None, str_template=None, include_attrs=None, exclude_attrs=None, no_verify=False, cache_path=None) +
    +
    +

    Returns an AccountLoader with accounts loaded from a URL.

    +

    Loaded accounts will include metadata associated with each account in the +document retrieved from the url. File based URLs can be used to load +data from a local file. This data will be parsed as JSON by default. To +override, use parser to specify a callable that accepts the text and +returns a list or dict of accounts (see MetaAccountLoader). To cache the +results, specify a non-zere number of seconds in max_age. +The default +location on disk is the system temp directory in a file called +awsrun.dat, which can be overrided via cache_path.

    +

    Given the following JSON:

    +
    {
+    "Accounts": [
+        {"id": "100200300400", "env": "prod", "status": "active"},
+        {"id": "200300400100", "env": "non-prod", "status": "active"},
+        {"id": "300400100200", "env": "non-prod", "status": "suspended"}
+    ]
+}
+
    +

    The account loader will build account objects with the following attribute +names: id, env, status. Assume the above JSON is returned from +http://example.com/accts.json:

    +
    loader = URLAccountLoader('http://example.com/accts.json', path=['Accounts'])
+accts = loader.accounts()
+
+# Let's inspect the 1st account object and its metadata
+assert accts[0].id == '100200300400'
+assert accts[0].env == 'prod'
+assert accts[0].status == 'active'
+
    +

    URLAccountLoader is a subclass of the MetaAccountLoader, which loads +accounts from a set of dicts. As such, the remainder of the parameters in +the constructor – id_attr, path, str_template, include_attrs, and +exclude_attrs – are defined in the constructor of MetaAccountLoader.

    +
    + +Expand source code + +
    class URLAccountLoader(MetaAccountLoader):
+    """Returns an `AccountLoader` with accounts loaded from a URL.
+
+    Loaded accounts will include metadata associated with each account in the
+    document retrieved from the `url`. File based URLs can be used to load
+    data from a local file. This data will be parsed as JSON by default. To
+    override, use `parser` to specify a callable that accepts the text and
+    returns a list or dict of accounts (see `MetaAccountLoader`). To cache the
+    results, specify a non-zere number of seconds in `max_age`.  The default
+    location on disk is the system temp directory in a file called
+    `awsrun.dat`, which can be overrided via `cache_path`.
+
+    Given the following JSON:
+
+        {
+            "Accounts": [
+                {"id": "100200300400", "env": "prod", "status": "active"},
+                {"id": "200300400100", "env": "non-prod", "status": "active"},
+                {"id": "300400100200", "env": "non-prod", "status": "suspended"}
+            ]
+        }
+
+    The account loader will build account objects with the following attribute
+    names: `id`, `env`, `status`. Assume the above JSON is returned from
+    http://example.com/accts.json:
+
+        loader = URLAccountLoader('http://example.com/accts.json', path=['Accounts'])
+        accts = loader.accounts()
+
+        # Let's inspect the 1st account object and its metadata
+        assert accts[0].id == '100200300400'
+        assert accts[0].env == 'prod'
+        assert accts[0].status == 'active'
+
+    URLAccountLoader is a subclass of the `MetaAccountLoader`, which loads
+    accounts from a set of dicts. As such, the remainder of the parameters in
+    the constructor -- `id_attr`, `path`, `str_template`, `include_attrs`, and
+    `exclude_attrs` -- are defined in the constructor of `MetaAccountLoader`.
+    """
+
+    def __init__(
+        self,
+        url,
+        parser=JSONParser(),
+        auth=None,
+        max_age=0,
+        id_attr="id",
+        path=None,
+        str_template=None,
+        include_attrs=None,
+        exclude_attrs=None,
+        no_verify=False,
+        cache_path=None,
+    ):
+
+        session = requests.Session()
+        session.mount("file://", FileAdapter())
+
+        def load_cache():
+            r = session.get(url, auth=auth, verify=not no_verify)
+            r.raise_for_status()
+            return parser(r.text)
+
+        if not cache_path:
+            cache_path = Path(tempfile.gettempdir(), "awsrun.dat")
+
+        accts = PersistentExpiringValue(load_cache, cache_path, max_age=max_age)
+
+        super().__init__(
+            accts.value(),
+            id_attr=id_attr,
+            path=[] if path is None else path,
+            str_template=str_template,
+            include_attrs=[] if include_attrs is None else include_attrs,
+            exclude_attrs=[] if exclude_attrs is None else exclude_attrs,
+        )
    +
    +

    Ancestors

    + +

    Inherited members

    + +
    class AbstractAccount (attributes) @@ -2951,6 +3409,21 @@

    AzureCLIAccountLoader

  • +

    CSVParser

    +
    • +
  • +

    JSONParser

    +
    • +
  • +

    YAMLParser

    +
    • +
  • +

    HTTPOAuth2

    +
    • +
  • +

    URLAccountLoader

    +
    • +

  • AbstractAccount

  • diff --git a/docs/cloudwatch.html b/docs/cloudwatch.html index 9fc1c08..40e7cb2 100644 --- a/docs/cloudwatch.html +++ b/docs/cloudwatch.html @@ -139,6 +139,7 @@

    Module awsrun.cloudwatch

    print(datetime, value) """ + import logging import math from collections import defaultdict diff --git a/docs/commands/aws/last.html b/docs/commands/aws/last.html index 6e7493d..567badc 100644 --- a/docs/commands/aws/last.html +++ b/docs/commands/aws/last.html @@ -984,6 +984,7 @@

    Command Options

    RowTable > DataTable { border: solid $accent-lighten-2; border-title-align: left; + height: 100%; } RowTable > DataTable:focus { border: solid $secondary; @@ -1711,7 +1712,21 @@

    Methods

    (prompt, help_md)
    • -

    An offscreen popup for a single input box.

    +

    An offscreen popup for a single input box.

    +

    Initialize a Widget.

    +

    Args

    +
    +
    *children
    +
    Child widgets.
    +
    name
    +
    The name of the widget.
    +
    id
    +
    The ID of the widget in the DOM.
    +
    classes
    +
    The CSS classes for the widget.
    +
    disabled
    +
    Whether the widget is disabled or not.
    +
    Expand source code @@ -1874,7 +1889,21 @@

    Example

    class FilterPopup
    -

    An offscreen popup for a single input box.

    +

    An offscreen popup for a single input box.

    +

    Initialize a Widget.

    +

    Args

    +
    +
    *children
    +
    Child widgets.
    +
    name
    +
    The name of the widget.
    +
    id
    +
    The ID of the widget in the DOM.
    +
    classes
    +
    The CSS classes for the widget.
    +
    disabled
    +
    Whether the widget is disabled or not.
    +
    Expand source code @@ -1925,7 +1954,21 @@

    Inherited members

    class ExportPopup
    -

    An offscreen popup for a single input box.

    +

    An offscreen popup for a single input box.

    +

    Initialize a Widget.

    +

    Args

    +
    +
    *children
    +
    Child widgets.
    +
    name
    +
    The name of the widget.
    +
    id
    +
    The ID of the widget in the DOM.
    +
    classes
    +
    The CSS classes for the widget.
    +
    disabled
    +
    Whether the widget is disabled or not.
    +
    Expand source code @@ -1977,7 +2020,21 @@

    Inherited members

    (*col_names)
    -

    Simple container widget, with vertical layout.

    +

    Simple container widget, with vertical layout.

    +

    Initialize a Widget.

    +

    Args

    +
    +
    *children
    +
    Child widgets.
    +
    name
    +
    The name of the widget.
    +
    id
    +
    The ID of the widget in the DOM.
    +
    classes
    +
    The CSS classes for the widget.
    +
    disabled
    +
    Whether the widget is disabled or not.
    +
    Expand source code @@ -1987,6 +2044,7 @@

    Inherited members

    RowTable > DataTable { border: solid $accent-lighten-2; border-title-align: left; + height: 100%; } RowTable > DataTable:focus { border: solid $secondary; @@ -2080,13 +2138,13 @@

    Args

    default
    A default value or callable that returns a default.
    layout
    -
    Perform a layout on change. Defaults to False.
    +
    Perform a layout on change.
    repaint
    -
    Perform a repaint on change. Defaults to True.
    +
    Perform a repaint on change.
    init
    -
    Call watchers on initialize (post mount). Defaults to True.
    +
    Call watchers on initialize (post mount).
    always_update
    -
    Call watchers even when the new value equals the old value. Defaults to False.
    +
    Call watchers even when the new value equals the old value.
    @@ -2159,9 +2217,10 @@

    Example

    Args

    scroll_visible
    -
    Scroll parent to make this widget -visible. Defaults to True.
    -
    +
    Scroll parent to make this widget visible.
    + +

    Returns

    +

    The Widget instance.

    Expand source code @@ -2191,7 +2250,21 @@

    Args

    class UserTable
    -

    Simple container widget, with vertical layout.

    +

    Simple container widget, with vertical layout.

    +

    Initialize a Widget.

    +

    Args

    +
    +
    *children
    +
    Child widgets.
    +
    name
    +
    The name of the widget.
    +
    id
    +
    The ID of the widget in the DOM.
    +
    classes
    +
    The CSS classes for the widget.
    +
    disabled
    +
    Whether the widget is disabled or not.
    +
    Expand source code @@ -2298,7 +2371,21 @@

    Inherited members

    class EventTable
    -

    Simple container widget, with vertical layout.

    +

    Simple container widget, with vertical layout.

    +

    Initialize a Widget.

    +

    Args

    +
    +
    *children
    +
    Child widgets.
    +
    name
    +
    The name of the widget.
    +
    id
    +
    The ID of the widget in the DOM.
    +
    classes
    +
    The CSS classes for the widget.
    +
    disabled
    +
    Whether the widget is disabled or not.
    +
    Expand source code @@ -2414,12 +2501,12 @@

    Inherited members

    Args

    driver_class
    -
    Driver class or None to auto-detect. Defaults to None.
    +
    Driver class or None to auto-detect. This will be used by some Textual tools.
    css_path
    -
    Path to CSS or None for no CSS file. -Defaults to None. To load multiple CSS files, pass a list of strings or paths which will be loaded in order.
    +
    Path to CSS or None to use the CSS_PATH class variable. +To load multiple CSS files, pass a list of strings or paths which will be loaded in order.
    watch_css
    -
    Watch CSS for changes. Defaults to False.
    +
    Reload CSS if the files changed. This is set automatically if you are using textual run with the dev switch.

    Raises

    @@ -2653,13 +2740,13 @@

    Args

    default
    A default value or callable that returns a default.
    layout
    -
    Perform a layout on change. Defaults to False.
    +
    Perform a layout on change.
    repaint
    -
    Perform a repaint on change. Defaults to True.
    +
    Perform a repaint on change.
    init
    -
    Call watchers on initialize (post mount). Defaults to True.
    +
    Call watchers on initialize (post mount).
    always_update
    -
    Call watchers even when the new value equals the old value. Defaults to False.
    +
    Call watchers even when the new value equals the old value.
    @@ -2690,13 +2777,13 @@

    Args

    default
    A default value or callable that returns a default.
    layout
    -
    Perform a layout on change. Defaults to False.
    +
    Perform a layout on change.
    repaint
    -
    Perform a repaint on change. Defaults to True.
    +
    Perform a repaint on change.
    init
    -
    Call watchers on initialize (post mount). Defaults to True.
    +
    Call watchers on initialize (post mount).
    always_update
    -
    Call watchers even when the new value equals the old value. Defaults to False.
    +
    Call watchers even when the new value equals the old value.
    @@ -2726,7 +2813,8 @@

    Methods

    def compose(self)
    -

    Yield child widgets for a container.

    +

    Yield child widgets for a container.

    +

    This method should be implemented in a subclass.

    Expand source code @@ -2793,7 +2881,7 @@

    Methods

    def action_toggle_dark(self)
    -

    Action to toggle dark mode.

    +

    An action to toggle dark mode.

    Expand source code @@ -3062,7 +3150,21 @@

    Methods

    class EventDetail
    -

    Simple container widget, with vertical layout.

    +

    Simple container widget, with vertical layout.

    +

    Initialize a Widget.

    +

    Args

    +
    +
    *children
    +
    Child widgets.
    +
    name
    +
    The name of the widget.
    +
    id
    +
    The ID of the widget in the DOM.
    +
    classes
    +
    The CSS classes for the widget.
    +
    disabled
    +
    Whether the widget is disabled or not.
    +
    Expand source code @@ -3158,13 +3260,13 @@

    Args

    default
    A default value or callable that returns a default.
    layout
    -
    Perform a layout on change. Defaults to False.
    +
    Perform a layout on change.
    repaint
    -
    Perform a repaint on change. Defaults to True.
    +
    Perform a repaint on change.
    init
    -
    Call watchers on initialize (post mount). Defaults to True.
    +
    Call watchers on initialize (post mount).
    always_update
    -
    Call watchers even when the new value equals the old value. Defaults to False.
    +
    Call watchers even when the new value equals the old value.
    diff --git a/docs/commands/aws/list_lambdas.html b/docs/commands/aws/list_lambdas.html index 5da5aff..c61e0a2 100644 --- a/docs/commands/aws/list_lambdas.html +++ b/docs/commands/aws/list_lambdas.html @@ -222,7 +222,7 @@

    Command Options

    total = len(by_role[role]) public = len([fn for fn in by_role[role] if _is_public(fn)]) print( - f"{acct}/{region}: role={role} total={total} private={total-public} public={public}", + f"{acct}/{region}: role={role} total={total} private={total - public} public={public}", file=out, ) @@ -296,7 +296,7 @@

    Classes

    total = len(by_role[role]) public = len([fn for fn in by_role[role] if _is_public(fn)]) print( - f"{acct}/{region}: role={role} total={total} private={total-public} public={public}", + f"{acct}/{region}: role={role} total={total} private={total - public} public={public}", file=out, ) diff --git a/docs/config.html b/docs/config.html index 0438655..01bf1c7 100644 --- a/docs/config.html +++ b/docs/config.html @@ -476,7 +476,7 @@

    Reading Values

    self.type = type_ def type_check(self, obj): - return type(obj) == self.type + return type(obj) == self.type # noqa: E721 def __str__(self): return self.type.__name__ @@ -492,7 +492,7 @@

    Reading Values

    self.pattern = pattern def type_check(self, obj): - if type(obj) != str: + if type(obj) != str: # noqa: E721 return False return bool(re.search(self.pattern, obj)) @@ -504,7 +504,7 @@

    Reading Values

    """Represents a string matching an IP address (v4 or v6).""" def type_check(self, obj): - if type(obj) != str: + if type(obj) != str: # noqa: E721 return False try: ipaddress.ip_address(obj) @@ -520,7 +520,7 @@

    Reading Values

    """Represents a string matching an IP network (v4 or v6).""" def type_check(self, obj): - if type(obj) != str: + if type(obj) != str: # noqa: E721 return False try: ipaddress.ip_network(obj) @@ -536,7 +536,7 @@

    Reading Values

    """Represents a string pointing to an existing file.""" def type_check(self, obj): - if type(obj) != str: + if type(obj) != str: # noqa: E721 return False return Path(obj).exists() @@ -600,7 +600,7 @@

    Reading Values

    self.element_type = element_type def type_check(self, obj): - if type(obj) != list: + if type(obj) != list: # noqa: E721 return False return all(self.element_type.type_check(e) for e in obj) @@ -623,7 +623,7 @@

    Reading Values

    self.value_type = value_type def type_check(self, obj): - if type(obj) != dict: + if type(obj) != dict: # noqa: E721 return False return all(self.key_type.type_check(k) for k in obj.keys()) and all( self.value_type.type_check(v) for v in obj.values() @@ -1357,7 +1357,7 @@

    Inherited members

    self.type = type_ def type_check(self, obj): - return type(obj) == self.type + return type(obj) == self.type # noqa: E721 def __str__(self): return self.type.__name__ @@ -1396,7 +1396,7 @@

    Inherited members

    self.pattern = pattern def type_check(self, obj): - if type(obj) != str: + if type(obj) != str: # noqa: E721 return False return bool(re.search(self.pattern, obj)) @@ -1429,7 +1429,7 @@

    Inherited members

    """Represents a string matching an IP address (v4 or v6).""" def type_check(self, obj): - if type(obj) != str: + if type(obj) != str: # noqa: E721 return False try: ipaddress.ip_address(obj) @@ -1466,7 +1466,7 @@

    Inherited members

    """Represents a string matching an IP network (v4 or v6).""" def type_check(self, obj): - if type(obj) != str: + if type(obj) != str: # noqa: E721 return False try: ipaddress.ip_network(obj) @@ -1503,7 +1503,7 @@

    Inherited members

    """Represents a string pointing to an existing file.""" def type_check(self, obj): - if type(obj) != str: + if type(obj) != str: # noqa: E721 return False return Path(obj).exists() @@ -1585,7 +1585,7 @@

    Inherited members

    self.element_type = element_type def type_check(self, obj): - if type(obj) != list: + if type(obj) != list: # noqa: E721 return False return all(self.element_type.type_check(e) for e in obj) @@ -1635,7 +1635,7 @@

    Inherited members

    self.value_type = value_type def type_check(self, obj): - if type(obj) != dict: + if type(obj) != dict: # noqa: E721 return False return all(self.key_type.type_check(k) for k in obj.keys()) and all( self.value_type.type_check(v) for v in obj.values() diff --git a/docs/index.html b/docs/index.html index 699d72d..fc1f0aa 100644 --- a/docs/index.html +++ b/docs/index.html @@ -316,7 +316,7 @@

    Roadmap

    """ name = "awsrun" -__version__ = "3.0.0" +__version__ = "3.1.0"
    diff --git a/docs/plugins/accts/index.html b/docs/plugins/accts/index.html index c7e8e8a..05adafb 100644 --- a/docs/plugins/accts/index.html +++ b/docs/plugins/accts/index.html @@ -156,14 +156,25 @@

    Module awsrun.plugins.accts

    `awsrun.plugmgr.Plugin` that returns a `awsrun.acctload.AccountLoader`. """ +import getpass +import os +from pathlib import Path + +from requests.auth import AuthBase, HTTPBasicAuth, HTTPDigestAuth +from requests_ntlm import HttpNtlmAuth + from awsrun.acctload import ( CSVAccountLoader, + CSVParser, + HTTPOAuth2, IdentityAccountLoader, JSONAccountLoader, + JSONParser, + URLAccountLoader, YAMLAccountLoader, + YAMLParser, ) -from awsrun.config import List, Str, Int, Bool, URL - +from awsrun.config import URL, Any, Bool, Choice, Dict, File, Int, List, Str from awsrun.plugmgr import Plugin @@ -719,7 +730,397 @@

    Module awsrun.plugins.accts

    no_verify=args.loader_no_verify, ) - return loader + return loader + + +class URLLoader(Plugin): + """CLI plug-in that loads accounts and metadata from a file/url. + + ## Overview + + Accounts specified on the awsrun CLI via the `--account` or + `--account-file` will be validated against the list of accounts loaded + from this URL. More importantly, loaded accounts will include metadata + associated with each account from the document. This metadata can be used + to select accounts using the `--include` and `--exclude` awsrun CLI flags. + Given the following JSON document: + + [ + {"id": "100200300400", "env": "prod", "status": "active"}, + {"id": "200300400100", "env": "non-prod", "status": "active"}, + {"id": "300400100200", "env": "non-prod", "status": "suspended"} + ] + + Users could select only the "active" accounts via the `awsrun.cli` by using + the metadata filter options. The following would select account numbers + "100200300400" and "200300400100": + + $ awsrun --include status=active aws ec2 describe-vpcs --region us-east-1 + + Additionally, this metadata is made available to command authors for use + within their commands. The account loader would build account objects with + the following attribute names: `id`, `env`, and `status`. Command authors + are provided access to these account objects in their user-defined commands + via a parameter to `awsrun.runner.Command.execute`: + + class CLICommand(Command): + def execute(self, session, acct): + # The acct parameter contains the attributes from the JSON + return f'{acct.env} account {acct.id} is {acct.status}\\n' + + In cases where the JSON key names are not valid Python identifiers, they are + munged. Leading digits are prefixed with underscores, non-alpha numeric + characters are replaced with underscores, and keywords are appended with an + underscore. + + Instead of specifying accounts as a JSON array of objects as shown above, + the JSON can be specified as a single object with account IDs as keys and + metadata as values such as: + + { + "100200300400": {"env": "prod", "status": "active"}, + "200300400100": {"env": "non-prod", "status": "active"}, + "300400100200": {"env": "non-prod", "status": "suspended"} + } + + The above will create the same list account objects as the first version + because this loader assumes a top-level JSON object is the container of + accounts. The account objects created will contain the `id` attribute + because the default value for the ID attribute is `id`. + + Similarly, the JSON can be specified as follows, where the account ID is + used in the top-level object key as well as the object value metadata: + + { + "100200300400": {"id": "100200300400", "env": "prod", "status": "active"}, + "200300400100": {"id": "200300400100", "env": "non-prod", "status": "active"}, + "300400100200": {"id": "300400100200", "env": "non-prod", "status": "suspended"} + } + + If the account list is not at the top-level of the JSON, a path can be + specified to point the loader to the correct location in the JSON. For + example, a path list of "aws" and "accounts" is required to parse the + following JSON: + + { + "aws": + { + "accounts": + [ + {"id": "100200300400", "env": "prod", "status": "active"}, + {"id": "200300400100", "env": "non-prod", "status": "active"}, + {"id": "300400100200", "env": "non-prod", "status": "suspended"} + ] + } + } + + The `URLLoader` can parse JSON, YAML, or CSV files loaded from a URL. Data + can be loaded from a local file via the "file:///" URL syntax. By default, + JSON is assumed. This can be overrided via the `parser` option. Parsing + options such as the CSV delimiter can be specified via `parser_options`. + When parsing CSV files, the header row is required as its names are the + key names for the metadata associated with each account. + + If the URL requires authentication, specify one of the supported types: + "none", "basic", "digest", "ntlm", or "oauth2" via the `auth` option. By + default, no authentication is used. Authentication options for the URL + such as username and password can be specified via `auth_options`. + + ## Configuration + + Options with an asterisk are mandatory and must be provided: + + Accounts: + plugin: awsrun.plugins.accts.URLLoader + options: + url: STRING* + auth: STRING ("none", "basic", "digest", "ntlm", "oauth2") + auth_options: + username: STRING + password: STRING + token_url: URL (oauth2 only) + scope: STRING (oauth2 only, default "AppIdClaimsTrust") + grant_type: STRING (oauth2 only, default "client_credentials") + parser: STRING ("json", "yaml", "csv") + parser_options: + delimiter: STRING (csv only, default ",") + max_age: INTEGER + id_attr: STRING* + path: + - STRING + str_template: STRING + include_attrs: + - STRING + exclude_attrs: + - STRING + no_verify: BOOLEAN + + ## Plug-in Options + + Some options can be overridden on the awsrun CLI via command line flags. + In those cases, the CLI flags are specified next to the option name below: + + `url`, `--loader-url` + : Load the data from the specified URL. To load a local file, use + `file:///absolute/path/data.json`. This value **must** be provided via the + user configuration or as an awsrun command line argument. + + `auth` + : Identifies the authentication type required by the URL. The default + value is "none". Valid options include "none", "basic", "digest", "ntml", + and "oauth2". All but "none" require that username` and `password` keys + in `auth_options`. In addition, "oauth2" requires `token_url` key. + + `auth_options` + : Provides options required for the specified `auth` type chosen. The use + of "basic", "digest", "ntml", or "oauth2" require `username` and + `password` keys in `auth_options`. Use of "oauth2" also requires a + `token_url` that points to the token provider. Three optional keys can be + provided with "oauth2" to override defaults: `scope` ("AppIdClaimsTrust"), + `grant_type` ("client_credentials"), or `intent` ("awsrun account loader + plugin"). + + `parser` + : Specifies the parser used to parse data from the URL. The default value + is "json". Valid options include "json", "yaml", and "csv". + + `parser_options` + : Provides additional options to the parser method used to parse the data + from the URL: `json.loads`, `yaml.safe_load`, and `csv.DictReader`. Most + often this would be used to change the `delimiter` used with the CSV + parser. + + `max_age`, `--loader-max-age` + : Cache the data retrieved from the URL for the specified number of seconds. + The default value is `0`, which disables caching. This can be useful for + servers that are slow to generate the account list. + + `id_attr` + : Identifies the JSON/YAML key name that contains the AWS account ID. This + value **must** be provided so awsrun can identify the account number + associated with each account in the JSON/YAML. + + `path` + : Specifies the location within the JSON/YAML that contains the array of + accounts or object of accounts. If specified, this must be a list of key + names to traverse the JSON/YAML. The default assumes the accounts are at + the top-level. + + `str_template`, `--loader-str-template` + : Controls how accounts are formatted as strings. This is a [Python format + string](https://docs.python.org/3.7/library/string.html#format-string-syntax) + that can include any of the included attributes. For example, `"{id}:{env}"` + or `"{id}-{env}"` assuming `id` and `env` are JSON/YAML/CSV key names. + + `include_attrs` + : Include only the specified list of JSON/YAML/CSV key names from the + object metadata. If this option is not supplied, all JSON/YAML/CSV keys + are included as attributes on the account objects created by this loader. + + `exclude_attrs` + : Exclude the specified list of JSON/YAML/CSV key names from the object + metadata. If this option is not supplied, no key names are excluded as + attributes on the account objects created by this loader. + + `no_verify`, `--loader-no-verify` + : Disable HTTP certificate verification. This is not advisable and user will + be warned on the command line if verification has been disabled. The default + value is `false`. + """ + + def __init__(self, parser, cfg): + super().__init__(parser, cfg) + + # Define the arguments that we want to allow a user to override via the + # main CLI. Any CLI args added via add_argument will be commingled with + # the main awsrun args, so they are prefixed with '--loader-' to lessen + # chance of collision. + group = parser.add_argument_group("account loader options") + group.add_argument( + "--loader-url", + metavar="URL", + default=cfg("url", type=URL, must_exist=True), + help="URL to account data (also supports file:///path/to/file)", + ) + + group.add_argument( + "--loader-auth", + metavar="STRING", + default=cfg( + "auth", + type=Choice("none", "basic", "digest", "ntml", "oauth2"), + default="none", + ), + help="Authentication type required by the URL", + ) + + group.add_argument( + "--loader-username", + metavar="USER", + help="username (or client_id) for account loader authentication", + ) + + group.add_argument( + "--loader-password", + metavar="PASS", + help="password (or client_secret) for account loader authentication", + ) + + group.add_argument( + "--loader-parser", + metavar="STRING", + default=cfg("parser", type=Choice("json", "yaml", "csv"), default="json"), + help="Parser used on data from URL", + ) + + group.add_argument( + "--loader-no-verify", + action="store_true", + default=cfg("no_verify", type=Bool, default=False), + help="disable cert verification for HTTP requests", + ) + + group.add_argument( + "--loader-cache-path", + metavar="PATH", + type=Path, + default=cfg("cache_path", type=File), + help="max age for cached URL data", + ) + + group.add_argument( + "--loader-max-age", + metavar="SECS", + type=int, + default=cfg("max_age", type=Int, default=0), + help="max age for cached URL data", + ) + + group.add_argument( + "--loader-str-template", + metavar="STRING", + default=cfg("str_template"), + help="format string used to display an account", + ) + + def instantiate(self, args): + cfg = self.cfg + + auth_options = cfg("auth_options", type=Dict(Str, Any), default={}) + + # Check and set auth options if using authentication. + if args.loader_auth != "none": + + # Command line flags take priority + if args.loader_username: + auth_options["username"] = args.loader_username + if args.loader_password: + auth_options["password"] = args.loader_password + + # If a username and password has not been provided via CLI flags + # or via the configuration file, we'll fallback to environment + # variables if they exist, or lastly we'll prompt the user + # interactively. BUT, we don't want to do that here because the + # URLAccountLoader caches data, so we might not need to make an + # HTTP call, and thus prompting the user would be unneeded (and + # annoying). So, instead, we use DeferPrompting to wrap the + # various requests HTTP*Auth classes. This will defer the + # instantiation of those classes until `requests` invokes the + # callable `auth` parameter to its various methods. + + if args.loader_auth == "oauth2" and "token_url" not in auth_options: + raise TypeError( + "with oauth2 authentication token_url must be set in config: Accounts->options->auth_options->token_url" + ) + + auth_types = { + "none": _HTTPNone(), + "basic": _DeferPrompting(HTTPBasicAuth, auth_options), + "digest": _DeferPrompting(HTTPDigestAuth, auth_options), + "ntlm": _DeferPrompting(HttpNtlmAuth, auth_options), + "oauth2": _DeferPrompting(HTTPOAuth2, auth_options), + } + auth = auth_types[args.loader_auth] + + parsers = { + "json": JSONParser, + "yaml": YAMLParser, + "csv": CSVParser, + } + parser_options = cfg("parser_options", type=Dict(Str, Any), default={}) + + try: + parser = parsers[args.loader_parser](**parser_options) + except TypeError as e: + raise TypeError( + f"incompatible parser_options specified in config: Accounts->options->parser_options: {e}" + ) + + loader = URLAccountLoader( + url=args.loader_url, + parser=parser, + auth=auth, + max_age=args.loader_max_age, + id_attr=cfg("id_attr", must_exist=True), + path=cfg("path", type=List(Str), default=[]), + str_template=args.loader_str_template, + include_attrs=cfg("include_attrs", type=List(Str), default=[]), + exclude_attrs=cfg("exclude_attrs", type=List(Str), default=[]), + no_verify=args.loader_no_verify, + cache_path=args.loader_cache_path, + ) + + return loader + + +def _default_username(): + return os.environ.get("AWSRUN_LOADER_USERNAME", None) or getpass.getuser() + + +def _default_password(user): + return os.environ.get("AWSRUN_LOADER_PASSWORD", None) or getpass.getpass( + f"Account loader password for {user}? " + ) + + +# Helper class to wrap one of `requests` auth classes to defer instantiation +# of those classes until `requests` actually needs to use the auth data. This +# is used to avoid interactively prompting a user for their password if one +# had not been specified via CLI args or their config file. The default will +# come from the env variable if set, otherwise the user is prompted. But we +# don't want to prompt when we instantiated the `requests` auth classes +# because at that time, we do not know if the account loader data has been +# cached and thus not require making an HTTP call. So, why bother prompting +# the user in that case? +class _DeferPrompting(AuthBase): + def __init__(self, auth_class, auth_options): + self.auth_class = auth_class + self.auth_options = auth_options + + def __call__(self, req): + if "username" not in self.auth_options: + self.auth_options["username"] = _default_username() + if "password" not in self.auth_options: + self.auth_options["password"] = _default_password( + self.auth_options["username"] + ) + try: + auth = self.auth_class(**self.auth_options) + except TypeError as e: + raise TypeError( + f"incompatible auth_options specified in config: Accounts->options->auth_options: {e}" + ) + return auth(req) + + +class _HTTPNone: + """HTTPNone is a no-op auth type for requests library.""" + + def __init__(self, *args, **kwargs): + return None + + def __call__(self, req): + return req
    @@ -1707,38 +2108,571 @@

    Inherited members

    - - - -