Skip to content

Commit

Permalink
Techdebt: Get regions from SSM instead of boto3
Browse files Browse the repository at this point in the history
  • Loading branch information
bblommers committed Oct 2, 2023
1 parent 3dcaa81 commit 65c25a9
Show file tree
Hide file tree
Showing 16 changed files with 125 additions and 32 deletions.
2 changes: 1 addition & 1 deletion moto/budgets/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,5 +141,5 @@ def describe_notifications_for_budget(


budgets_backends = BackendDict(
BudgetsBackend, "budgets", use_boto3_regions=False, additional_regions=["global"]
BudgetsBackend, "budgets", use_ssm_regions=False, additional_regions=["global"]
)
2 changes: 1 addition & 1 deletion moto/ce/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,5 +153,5 @@ def untag_resource(self, resource_arn: str, tag_keys: List[str]) -> None:


ce_backends = BackendDict(
CostExplorerBackend, "ce", use_boto3_regions=False, additional_regions=["global"]
CostExplorerBackend, "ce", use_ssm_regions=False, additional_regions=["global"]
)
2 changes: 1 addition & 1 deletion moto/cloudfront/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -432,6 +432,6 @@ def delete_origin_access_control(self, control_id: str) -> None:
cloudfront_backends = BackendDict(
CloudFrontBackend,
"cloudfront",
use_boto3_regions=False,
use_ssm_regions=False,
additional_regions=["global"],
)
21 changes: 10 additions & 11 deletions moto/core/base_backend.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
from boto3 import Session
import re
import string
from functools import lru_cache
from threading import RLock
from typing import Any, List, Dict, Optional, ClassVar, TypeVar, Iterator
from uuid import uuid4
from moto.settings import allow_unknown_region, enable_iso_regions
from moto.utilities.utils import load_resource
from .model_instances import model_data
from .utils import convert_regex_to_flask_path


regions_by_service = load_resource(__name__, "regions.json")


class InstanceTrackerMeta(type):
def __new__(meta, name: str, bases: Any, dct: Dict[str, Any]) -> type:
cls = super(InstanceTrackerMeta, meta).__new__(meta, name, bases, dct)
Expand Down Expand Up @@ -191,19 +194,15 @@ def __init__(
service_name: str,
account_id: str,
backend: type,
use_boto3_regions: bool,
use_ssm_regions: bool,
additional_regions: Optional[List[str]],
):
self.service_name = service_name
self.account_id = account_id
self.backend = backend
self.regions = []
if use_boto3_regions:
sess = Session()
for partition in sess.get_available_partitions():
self.regions.extend(
sess.get_available_regions(service_name, partition_name=partition)
)
if use_ssm_regions:
self.regions.extend(regions_by_service.get(service_name) or regions_by_service["__all"])
self.regions.extend(additional_regions or [])
self._id = str(uuid4())

Expand Down Expand Up @@ -268,12 +267,12 @@ def __init__(
self,
backend: Any,
service_name: str,
use_boto3_regions: bool = True,
use_ssm_regions: bool = True,
additional_regions: Optional[List[str]] = None,
):
self.backend = backend
self.service_name = service_name
self._use_boto3_regions = use_boto3_regions
self._use_ssm_regions = use_ssm_regions
self._additional_regions = additional_regions
self._id = str(uuid4())

Expand Down Expand Up @@ -312,6 +311,6 @@ def _create_account_specific_backend(self, account_id: str) -> None:
service_name=self.service_name,
account_id=account_id,
backend=self.backend,
use_boto3_regions=self._use_boto3_regions,
use_ssm_regions=self._use_ssm_regions,
additional_regions=self._additional_regions,
)
59 changes: 59 additions & 0 deletions moto/core/regions.json

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion moto/dynamodb_v20111205/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,6 @@ def update_item(
dynamodb_backends = BackendDict(
DynamoDBBackend,
"dynamodb_v20111205",
use_boto3_regions=False,
use_ssm_regions=False,
additional_regions=["global"],
)
2 changes: 1 addition & 1 deletion moto/iam/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -3310,5 +3310,5 @@ def get_service_linked_role_deletion_status(self) -> bool:


iam_backends = BackendDict(
IAMBackend, "iam", use_boto3_regions=False, additional_regions=["global"]
IAMBackend, "iam", use_ssm_regions=False, additional_regions=["global"]
)
2 changes: 1 addition & 1 deletion moto/instance_metadata/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ class InstanceMetadataBackend(BaseBackend):
instance_metadata_backends = BackendDict(
InstanceMetadataBackend,
"instance_metadata",
use_boto3_regions=False,
use_ssm_regions=False,
additional_regions=["global"],
)
2 changes: 1 addition & 1 deletion moto/organizations/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,6 @@ def remove_account_from_organization(self, **kwargs: str) -> None:
organizations_backends = BackendDict(
OrganizationsBackend,
"organizations",
use_boto3_regions=False,
use_ssm_regions=False,
additional_regions=["global"],
)
2 changes: 1 addition & 1 deletion moto/route53/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -929,5 +929,5 @@ def get_reusable_delegation_set(self, delegation_set_id: str) -> DelegationSet:


route53_backends = BackendDict(
Route53Backend, "route53", use_boto3_regions=False, additional_regions=["global"]
Route53Backend, "route53", use_ssm_regions=False, additional_regions=["global"]
)
6 changes: 3 additions & 3 deletions moto/s3/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2619,16 +2619,16 @@ def __init__(
self,
backend: Any,
service_name: str,
use_boto3_regions: bool = True,
use_ssm_regions: bool = True,
additional_regions: Optional[List[str]] = None,
):
super().__init__(backend, service_name, use_boto3_regions, additional_regions)
super().__init__(backend, service_name, use_ssm_regions, additional_regions)

# Maps bucket names to account IDs. This is used to locate the exact S3Backend
# holding the bucket and to maintain the common bucket namespace.
self.bucket_accounts: Dict[str, str] = {}


s3_backends = S3BackendDict(
S3Backend, service_name="s3", use_boto3_regions=False, additional_regions=["global"]
S3Backend, service_name="s3", use_ssm_regions=False, additional_regions=["global"]
)
2 changes: 1 addition & 1 deletion moto/s3control/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,6 @@ def get_access_point_policy_status(self, account_id: str, name: str) -> bool:
s3control_backends = BackendDict(
S3ControlBackend,
"s3control",
use_boto3_regions=False,
use_ssm_regions=False,
additional_regions=["global"],
)
2 changes: 1 addition & 1 deletion moto/sts/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -198,5 +198,5 @@ def _create_access_key(self, role: str) -> Tuple[str, AccessKey]:


sts_backends = BackendDict(
STSBackend, "sts", use_boto3_regions=False, additional_regions=["global"]
STSBackend, "sts", use_ssm_regions=False, additional_regions=["global"]
)
2 changes: 1 addition & 1 deletion moto/support/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,5 +224,5 @@ def describe_cases(


support_backends = BackendDict(
SupportBackend, "support", use_boto3_regions=False, additional_regions=["us-east-1"]
SupportBackend, "support", use_ssm_regions=False, additional_regions=["us-east-1"]
)
45 changes: 40 additions & 5 deletions scripts/ssm_get_default_params.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,23 @@
import subprocess
import time

import moto
from moto.utilities.utils import load_resource
from moto.ssm.utils import convert_to_tree


root_dir = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode().strip()


alt_names = {
"lambda": "awslambda",
"aps": "amp", # Prometheus
"costexplorer": "ce",
"waf": "wafv2",
"identitystore": "ssoadmin"
}


def retrieve_by_path(client, path):
print(f"Retrieving all parameters from {path}. "
f"AWS has around 14000 parameters, and we can only retrieve 10 at the time, so this may take a while.\n\n")
Expand Down Expand Up @@ -44,17 +58,38 @@ def main():
for path in default_param_paths:
params = retrieve_by_path(client, path)
tree = convert_to_tree(params)
root_dir = (
subprocess.check_output(["git", "rev-parse", "--show-toplevel"])
.decode()
.strip()
)

filename = "{}.json".format(path.split("/")[-1])
dest = os.path.join(root_dir, "moto/ssm/resources/{}".format(filename))
print("Writing data to {0}".format(dest))
with open(dest, "w") as open_file:
json.dump(tree, open_file, sort_keys=True, indent=2)

services = load_resource(moto.__name__, "ssm/resources/services.json")["aws"]["service"]["global-infrastructure"]["services"]
all_regions = set()
for service in services.keys():
all_regions.update(list(services[service].get("regions", {}).keys()))

regions_by_service = {"__all": list(all_regions)}
for service in services.keys():
regions = set(services[service].get("regions", {}).keys())
if regions != set(regions_by_service["__all"]):
clean_name = alt_names.get(service, service).replace("-", "")
if f"mock_{clean_name}" in dir(moto):
regions_by_service[service] = sorted(regions)

dest = os.path.join(root_dir, "moto/core/regions.json")
print("Writing data to {0}".format(dest))
with open(dest, "w") as open_file:
# Yes, json.dump exists
# But this gives us more control over the indentation options
open_file.write("{\n")
for idx, key in enumerate(sorted(regions_by_service.keys())):
open_file.write(f" \"{key}\": [")
open_file.write(", ".join([f"\"{s}\"" for s in sorted(regions_by_service[key])]))
open_file.write("]\n" if (idx == len(regions_by_service.keys())-1) else "],\n")
open_file.write("}")


if __name__ == "__main__":
main()
4 changes: 2 additions & 2 deletions tests/test_core/test_backenddict.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def test_backend_dict_can_retrieve_for_specific_account():


def test_backend_dict_can_ignore_boto3_regions():
backend_dict = BackendDict(ExampleBackend, "ec2", use_boto3_regions=False)
backend_dict = BackendDict(ExampleBackend, "ec2", use_ssm_regions=False)
assert backend_dict["account"].get("us-east-1") is None


Expand Down Expand Up @@ -143,7 +143,7 @@ def _create_asb(account_id, backend=None, use_boto3_regions=False, regions=None)
service_name="ec2",
account_id=account_id,
backend=backend or ExampleBackend,
use_boto3_regions=use_boto3_regions,
use_ssm_regions=use_boto3_regions,
additional_regions=regions,
)

Expand Down

0 comments on commit 65c25a9

Please sign in to comment.