From 768b38012f735a35bb6410ca8c59190a9453c059 Mon Sep 17 00:00:00 2001 From: George Campbell Date: Fri, 25 Oct 2024 17:23:08 -0700 Subject: [PATCH 1/2] upgrade to pydantic 2 --- .pylintrc | 2 ++ service_capacity_modeling/capacity_planner.py | 25 ++++++++++++------- service_capacity_modeling/interface.py | 6 ++--- service_capacity_modeling/models/common.py | 4 +-- .../models/org/netflix/aurora.py | 2 +- .../models/org/netflix/cassandra.py | 4 +-- .../models/org/netflix/crdb.py | 4 +-- .../models/org/netflix/elasticsearch.py | 2 +- .../models/org/netflix/evcache.py | 2 +- .../models/org/netflix/kafka.py | 2 +- .../models/org/netflix/rds.py | 2 +- .../models/org/netflix/stateless_java.py | 2 +- setup.py | 2 +- tests/netflix/test_cassandra.py | 4 +-- tox.ini | 3 ++- 15 files changed, 38 insertions(+), 28 deletions(-) diff --git a/.pylintrc b/.pylintrc index 990face..3fb1aae 100644 --- a/.pylintrc +++ b/.pylintrc @@ -9,6 +9,8 @@ extension-pkg-whitelist=pydantic.fields ignored-argument-names=args|kwargs disable = + # in python 3.8 of expected a valid pylint message and got 'too-many-positional-arguments' + unknown-option-value, # often useful to be explicit on which returns should return None useless-return, # gets angry when you declare a module-level logger diff --git a/service_capacity_modeling/capacity_planner.py b/service_capacity_modeling/capacity_planner.py index aa08a1f..fabde6b 100644 --- a/service_capacity_modeling/capacity_planner.py +++ b/service_capacity_modeling/capacity_planner.py @@ -355,7 +355,7 @@ def models(self) -> Dict[str, CapacityModel]: def hardware_shapes(self) -> HardwareShapes: return self._shapes - def _plan_percentiles( + def _plan_percentiles( # pylint: disable=too-many-positional-arguments self, model_name: str, percentiles: Tuple[int, ...], @@ -421,7 +421,7 @@ def _plan_percentiles( return mean_plan, percentile_plans - def _group_plans_by_percentile( + def _group_plans_by_percentile( # pylint: disable=too-many-positional-arguments self, drives, extra_model_arguments, @@ -462,7 +462,7 @@ def _group_plans_by_percentile( ) return percentile_plans - def _mean_plan( + def _mean_plan( # pylint: disable=too-many-positional-arguments self, drives, extra_model_arguments, @@ -494,7 +494,7 @@ def _mean_plan( ) return mean_plan - def plan_certain( + def plan_certain( # pylint: disable=too-many-positional-arguments self, model_name: str, region: str, @@ -538,7 +538,7 @@ def plan_certain( return [functools.reduce(merge_plan, composed) for composed in zip(*results)] - def _plan_certain( + def _plan_certain( # pylint: disable=too-many-positional-arguments self, model_name: str, region: str, @@ -573,8 +573,15 @@ def _plan_certain( num_results = num_results or self._default_num_results return reduce_by_family(plans)[:num_results] - def generate_scenarios( - self, model, region, desires, num_regions, lifecycles, instance_families, drives + def generate_scenarios( # pylint: disable=too-many-positional-arguments + self, + model, + region, + desires, + num_regions, + lifecycles, + instance_families, + drives, ): lifecycles = lifecycles or self._default_lifecycles instance_families = instance_families or [] @@ -638,8 +645,8 @@ def generate_scenarios( drive = Drive.get_managed_drive() yield instance, drive, context - # pylint: disable-msg=too-many-locals - def plan( + # pylint: disable=too-many-locals + def plan( # pylint: disable=too-many-positional-arguments self, model_name: str, region: str, diff --git a/service_capacity_modeling/interface.py b/service_capacity_modeling/interface.py index c97455e..7e8b79c 100644 --- a/service_capacity_modeling/interface.py +++ b/service_capacity_modeling/interface.py @@ -67,8 +67,8 @@ class Interval(ExcludeUnsetModel): maximum_value: Optional[float] = None class Config: - allow_mutation = False frozen = True + protected_namespaces = () @property def can_simulate(self): @@ -267,7 +267,7 @@ def annual_cost(self): # (160000.0, 0.384)] r_cost, w_cost, offset = 0.0, 0.0, 0.0 if self.annual_cost_per_read_io: - for (end, cost) in self.annual_cost_per_read_io: + for end, cost in self.annual_cost_per_read_io: charge_ios = min(r_ios, end) - offset r_cost += charge_ios * cost offset += charge_ios @@ -276,7 +276,7 @@ def annual_cost(self): offset = 0.0 if self.annual_cost_per_write_io: - for (end, cost) in self.annual_cost_per_write_io: + for end, cost in self.annual_cost_per_write_io: charge_ios = min(w_ios, end) - offset w_cost += charge_ios * cost offset += charge_ios diff --git a/service_capacity_modeling/models/common.py b/service_capacity_modeling/models/common.py index ddb4b1a..2f98b34 100644 --- a/service_capacity_modeling/models/common.py +++ b/service_capacity_modeling/models/common.py @@ -158,7 +158,7 @@ def network_services( return result -def compute_stateless_region( +def compute_stateless_region( # pylint: disable=too-many-positional-arguments instance: Instance, needed_cores: int, needed_memory_gib: float, @@ -200,7 +200,7 @@ def compute_stateless_region( # pylint: disable=too-many-locals # pylint: disable=too-many-statements -def compute_stateful_zone( +def compute_stateful_zone( # pylint: disable=too-many-positional-arguments instance: Instance, drive: Drive, needed_cores: int, diff --git a/service_capacity_modeling/models/org/netflix/aurora.py b/service_capacity_modeling/models/org/netflix/aurora.py index dbecf76..228a8d9 100644 --- a/service_capacity_modeling/models/org/netflix/aurora.py +++ b/service_capacity_modeling/models/org/netflix/aurora.py @@ -124,7 +124,7 @@ def _estimate_io_cost( return r_cost + w_cost -def _compute_aurora_region( +def _compute_aurora_region( # pylint: disable=too-many-positional-arguments instance: Instance, drive: Drive, # always to be Aurora Storage needed_cores: int, diff --git a/service_capacity_modeling/models/org/netflix/cassandra.py b/service_capacity_modeling/models/org/netflix/cassandra.py index 4661445..8deb8ae 100644 --- a/service_capacity_modeling/models/org/netflix/cassandra.py +++ b/service_capacity_modeling/models/org/netflix/cassandra.py @@ -62,7 +62,7 @@ def _write_buffer_gib_zone( return float(write_buffer_gib) / zones_per_region -def _estimate_cassandra_requirement( +def _estimate_cassandra_requirement( # pylint: disable=too-many-positional-arguments instance: Instance, desires: CapacityDesires, working_set: float, @@ -204,7 +204,7 @@ def _upsert_params(cluster, params): # pylint: disable=too-many-locals # pylint: disable=too-many-return-statements # flake8: noqa: C901 -def _estimate_cassandra_cluster_zonal( +def _estimate_cassandra_cluster_zonal( # pylint: disable=too-many-positional-arguments instance: Instance, drive: Drive, context: RegionContext, diff --git a/service_capacity_modeling/models/org/netflix/crdb.py b/service_capacity_modeling/models/org/netflix/crdb.py index cf9b727..ff1fa09 100644 --- a/service_capacity_modeling/models/org/netflix/crdb.py +++ b/service_capacity_modeling/models/org/netflix/crdb.py @@ -46,7 +46,7 @@ def _crdb_io_per_read(node_size_gib, sstable_size_mb=1000): return levels -def _estimate_cockroachdb_requirement( +def _estimate_cockroachdb_requirement( # noqa=E501 pylint: disable=too-many-positional-arguments instance: Instance, desires: CapacityDesires, working_set: float, @@ -129,7 +129,7 @@ def _upsert_params(cluster, params): # pylint: disable=too-many-locals -def _estimate_cockroachdb_cluster_zonal( +def _estimate_cockroachdb_cluster_zonal( # noqa=E501 pylint: disable=too-many-positional-arguments instance: Instance, drive: Drive, desires: CapacityDesires, diff --git a/service_capacity_modeling/models/org/netflix/elasticsearch.py b/service_capacity_modeling/models/org/netflix/elasticsearch.py index 1927962..4888d58 100644 --- a/service_capacity_modeling/models/org/netflix/elasticsearch.py +++ b/service_capacity_modeling/models/org/netflix/elasticsearch.py @@ -60,7 +60,7 @@ def _es_io_per_read(node_size_gib, segment_size_mb=512): return levels -def _estimate_elasticsearch_requirement( +def _estimate_elasticsearch_requirement( # noqa: E501 pylint: disable=too-many-positional-arguments node_type: str, instance: Instance, desires: CapacityDesires, diff --git a/service_capacity_modeling/models/org/netflix/evcache.py b/service_capacity_modeling/models/org/netflix/evcache.py index 754acf1..43d90d5 100644 --- a/service_capacity_modeling/models/org/netflix/evcache.py +++ b/service_capacity_modeling/models/org/netflix/evcache.py @@ -160,7 +160,7 @@ def _upsert_params(cluster, params): # pylint: disable=too-many-locals -def _estimate_evcache_cluster_zonal( # noqa: C901 +def _estimate_evcache_cluster_zonal( # noqa: C901,E501 pylint: disable=too-many-positional-arguments instance: Instance, drive: Drive, desires: CapacityDesires, diff --git a/service_capacity_modeling/models/org/netflix/kafka.py b/service_capacity_modeling/models/org/netflix/kafka.py index 218ff0b..00a3fd0 100644 --- a/service_capacity_modeling/models/org/netflix/kafka.py +++ b/service_capacity_modeling/models/org/netflix/kafka.py @@ -151,7 +151,7 @@ def _kafka_read_io(rps, io_size_kib, size_gib, recovery_seconds: int) -> float: # pylint: disable=too-many-locals # pylint: disable=too-many-return-statements -def _estimate_kafka_cluster_zonal( +def _estimate_kafka_cluster_zonal( # pylint: disable=too-many-positional-arguments instance: Instance, drive: Drive, desires: CapacityDesires, diff --git a/service_capacity_modeling/models/org/netflix/rds.py b/service_capacity_modeling/models/org/netflix/rds.py index b645d20..d47b6f0 100644 --- a/service_capacity_modeling/models/org/netflix/rds.py +++ b/service_capacity_modeling/models/org/netflix/rds.py @@ -91,7 +91,7 @@ def _rds_required_disk_ios(disk_size_gib: int, db_type: str, btree_fan_out: int return math.log(pages, btree_fan_out) -def _compute_rds_region( +def _compute_rds_region( # pylint: disable=too-many-positional-arguments instance: Instance, drive: Drive, needed_cores: int, diff --git a/service_capacity_modeling/models/org/netflix/stateless_java.py b/service_capacity_modeling/models/org/netflix/stateless_java.py index fe38d53..16d125c 100644 --- a/service_capacity_modeling/models/org/netflix/stateless_java.py +++ b/service_capacity_modeling/models/org/netflix/stateless_java.py @@ -73,7 +73,7 @@ def _estimate_java_app_requirement( ) -def _estimate_java_app_region( +def _estimate_java_app_region( # pylint: disable=too-many-positional-arguments instance: Instance, drive: Drive, desires: CapacityDesires, diff --git a/setup.py b/setup.py index 15d05e8..ffd7dc2 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,7 @@ description="Contains utilities for modeling database capacity on a cloud", packages=setuptools.find_packages(exclude=("tests*", "notebooks*")), install_requires=[ - "pydantic<2.0", + "pydantic>2.0", "scipy", "numpy", 'importlib_resources; python_version < "3.7"', diff --git a/tests/netflix/test_cassandra.py b/tests/netflix/test_cassandra.py index ba20089..c176b61 100644 --- a/tests/netflix/test_cassandra.py +++ b/tests/netflix/test_cassandra.py @@ -5,8 +5,8 @@ from service_capacity_modeling.interface import certain_float from service_capacity_modeling.interface import certain_int from service_capacity_modeling.interface import Consistency -from service_capacity_modeling.interface import CurrentClusterCapacity from service_capacity_modeling.interface import CurrentClusters +from service_capacity_modeling.interface import CurrentZoneClusterCapacity from service_capacity_modeling.interface import DataShape from service_capacity_modeling.interface import FixedInterval from service_capacity_modeling.interface import GlobalConsistency @@ -311,7 +311,7 @@ def test_plan_certain(): Use cpu utilization to determine instance types directly as supposed to extrapolating it from the Data Shape """ - cluster_capacity = CurrentClusterCapacity( + cluster_capacity = CurrentZoneClusterCapacity( cluster_instance_name="i4i.8xlarge", cluster_instance_count=Interval(low=8, mid=8, high=8, confidence=1), cpu_utilization=Interval( diff --git a/tox.ini b/tox.ini index 357262e..c34c816 100644 --- a/tox.ini +++ b/tox.ini @@ -26,10 +26,11 @@ skip_install = true deps = pre-commit pylint - pydantic<2.0 + pydantic>2.0 scipy numpy isodate + setuptools commands = pre-commit run --all-files From 3722c2bba8fb605b21f78370c73316af40a55335 Mon Sep 17 00:00:00 2001 From: George Campbell Date: Mon, 28 Oct 2024 10:08:26 -0700 Subject: [PATCH 2/2] run bump-pydantic --- service_capacity_modeling/interface.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/service_capacity_modeling/interface.py b/service_capacity_modeling/interface.py index 7e8b79c..e2bdee2 100644 --- a/service_capacity_modeling/interface.py +++ b/service_capacity_modeling/interface.py @@ -16,6 +16,7 @@ import numpy as np from pydantic import BaseModel +from pydantic import ConfigDict from pydantic import Field GIB_IN_BYTES = 1024 * 1024 * 1024 @@ -65,10 +66,7 @@ class Interval(ExcludeUnsetModel): minimum_value: Optional[float] = None maximum_value: Optional[float] = None - - class Config: - frozen = True - protected_namespaces = () + model_config = ConfigDict(frozen=True, protected_namespaces=()) @property def can_simulate(self): @@ -318,7 +316,7 @@ class Instance(ExcludeUnsetModel): cpu_ghz: float ram_gib: float net_mbps: float - drive: Optional[Drive] + drive: Optional[Drive] = None annual_cost: float = 0 lifecycle: Lifecycle = Lifecycle.stable # Typically hardware has a single platform, but sometimes they can act in multiple