Merge pull request #93 from Netflix-Skunkworks/pydantic-v2

upgrade to pydantic 2

.pylintrc

@@ -9,6 +9,8 @@ extension-pkg-whitelist=pydantic.fields
 ignored-argument-names=args|kwargs
 
 disable =
+  # in python 3.8 of expected a valid pylint message and got 'too-many-positional-arguments'
+  unknown-option-value,
   # often useful to be explicit on which returns should return None
   useless-return,
   # gets angry when you declare a module-level logger

service_capacity_modeling/capacity_planner.py

@@ -355,7 +355,7 @@ def models(self) -> Dict[str, CapacityModel]:
     def hardware_shapes(self) -> HardwareShapes:
         return self._shapes
 
-    def _plan_percentiles(
+    def _plan_percentiles(  # pylint: disable=too-many-positional-arguments
         self,
         model_name: str,
         percentiles: Tuple[int, ...],
@@ -421,7 +421,7 @@ def _plan_percentiles(
 
         return mean_plan, percentile_plans
 
-    def _group_plans_by_percentile(
+    def _group_plans_by_percentile(  # pylint: disable=too-many-positional-arguments
         self,
         drives,
         extra_model_arguments,
@@ -462,7 +462,7 @@ def _group_plans_by_percentile(
             )
         return percentile_plans
 
-    def _mean_plan(
+    def _mean_plan(  # pylint: disable=too-many-positional-arguments
         self,
         drives,
         extra_model_arguments,
@@ -494,7 +494,7 @@ def _mean_plan(
         )
         return mean_plan
 
-    def plan_certain(
+    def plan_certain(  # pylint: disable=too-many-positional-arguments
         self,
         model_name: str,
         region: str,
@@ -538,7 +538,7 @@ def plan_certain(
 
         return [functools.reduce(merge_plan, composed) for composed in zip(*results)]
 
-    def _plan_certain(
+    def _plan_certain(  # pylint: disable=too-many-positional-arguments
         self,
         model_name: str,
         region: str,
@@ -573,8 +573,15 @@ def _plan_certain(
         num_results = num_results or self._default_num_results
         return reduce_by_family(plans)[:num_results]
 
-    def generate_scenarios(
-        self, model, region, desires, num_regions, lifecycles, instance_families, drives
+    def generate_scenarios(  # pylint: disable=too-many-positional-arguments
+        self,
+        model,
+        region,
+        desires,
+        num_regions,
+        lifecycles,
+        instance_families,
+        drives,
     ):
         lifecycles = lifecycles or self._default_lifecycles
         instance_families = instance_families or []
@@ -638,8 +645,8 @@ def generate_scenarios(
             drive = Drive.get_managed_drive()
             yield instance, drive, context
 
-    # pylint: disable-msg=too-many-locals
-    def plan(
+    # pylint: disable=too-many-locals
+    def plan(  # pylint: disable=too-many-positional-arguments
         self,
         model_name: str,
         region: str,

service_capacity_modeling/interface.py

@@ -16,6 +16,7 @@
 
 import numpy as np
 from pydantic import BaseModel
+from pydantic import ConfigDict
 from pydantic import Field
 
 GIB_IN_BYTES = 1024 * 1024 * 1024
@@ -65,10 +66,7 @@ class Interval(ExcludeUnsetModel):
 
     minimum_value: Optional[float] = None
     maximum_value: Optional[float] = None
-
-    class Config:
-        allow_mutation = False
-        frozen = True
+    model_config = ConfigDict(frozen=True, protected_namespaces=())
 
     @property
     def can_simulate(self):
@@ -267,7 +265,7 @@ def annual_cost(self):
         #  (160000.0, 0.384)]
         r_cost, w_cost, offset = 0.0, 0.0, 0.0
         if self.annual_cost_per_read_io:
-            for (end, cost) in self.annual_cost_per_read_io:
+            for end, cost in self.annual_cost_per_read_io:
                 charge_ios = min(r_ios, end) - offset
                 r_cost += charge_ios * cost
                 offset += charge_ios
@@ -276,7 +274,7 @@ def annual_cost(self):
 
         offset = 0.0
         if self.annual_cost_per_write_io:
-            for (end, cost) in self.annual_cost_per_write_io:
+            for end, cost in self.annual_cost_per_write_io:
                 charge_ios = min(w_ios, end) - offset
                 w_cost += charge_ios * cost
                 offset += charge_ios
@@ -318,7 +316,7 @@ class Instance(ExcludeUnsetModel):
     cpu_ghz: float
     ram_gib: float
     net_mbps: float
-    drive: Optional[Drive]
+    drive: Optional[Drive] = None
     annual_cost: float = 0
     lifecycle: Lifecycle = Lifecycle.stable
     # Typically hardware has a single platform, but sometimes they can act in multiple

service_capacity_modeling/models/common.py

@@ -158,7 +158,7 @@ def network_services(
     return result
 
 
-def compute_stateless_region(
+def compute_stateless_region(  # pylint: disable=too-many-positional-arguments
     instance: Instance,
     needed_cores: int,
     needed_memory_gib: float,
@@ -200,7 +200,7 @@ def compute_stateless_region(
 
 # pylint: disable=too-many-locals
 # pylint: disable=too-many-statements
-def compute_stateful_zone(
+def compute_stateful_zone(  # pylint: disable=too-many-positional-arguments
     instance: Instance,
     drive: Drive,
     needed_cores: int,

service_capacity_modeling/models/org/netflix/aurora.py

@@ -124,7 +124,7 @@ def _estimate_io_cost(
     return r_cost + w_cost
 
 
-def _compute_aurora_region(
+def _compute_aurora_region(  # pylint: disable=too-many-positional-arguments
     instance: Instance,
     drive: Drive,  # always to be Aurora Storage
     needed_cores: int,

service_capacity_modeling/models/org/netflix/cassandra.py

@@ -62,7 +62,7 @@ def _write_buffer_gib_zone(
     return float(write_buffer_gib) / zones_per_region
 
 
-def _estimate_cassandra_requirement(
+def _estimate_cassandra_requirement(  # pylint: disable=too-many-positional-arguments
     instance: Instance,
     desires: CapacityDesires,
     working_set: float,
@@ -204,7 +204,7 @@ def _upsert_params(cluster, params):
 # pylint: disable=too-many-locals
 # pylint: disable=too-many-return-statements
 # flake8: noqa: C901
-def _estimate_cassandra_cluster_zonal(
+def _estimate_cassandra_cluster_zonal(  # pylint: disable=too-many-positional-arguments
     instance: Instance,
     drive: Drive,
     context: RegionContext,

service_capacity_modeling/models/org/netflix/crdb.py

@@ -46,7 +46,7 @@ def _crdb_io_per_read(node_size_gib, sstable_size_mb=1000):
     return levels
 
 
-def _estimate_cockroachdb_requirement(
+def _estimate_cockroachdb_requirement(  # noqa=E501 pylint: disable=too-many-positional-arguments
     instance: Instance,
     desires: CapacityDesires,
     working_set: float,
@@ -129,7 +129,7 @@ def _upsert_params(cluster, params):
 
 
 # pylint: disable=too-many-locals
-def _estimate_cockroachdb_cluster_zonal(
+def _estimate_cockroachdb_cluster_zonal(  # noqa=E501 pylint: disable=too-many-positional-arguments
     instance: Instance,
     drive: Drive,
     desires: CapacityDesires,

service_capacity_modeling/models/org/netflix/elasticsearch.py

@@ -60,7 +60,7 @@ def _es_io_per_read(node_size_gib, segment_size_mb=512):
     return levels
 
 
-def _estimate_elasticsearch_requirement(
+def _estimate_elasticsearch_requirement(  # noqa: E501 pylint: disable=too-many-positional-arguments
     node_type: str,
     instance: Instance,
     desires: CapacityDesires,

service_capacity_modeling/models/org/netflix/evcache.py

@@ -160,7 +160,7 @@ def _upsert_params(cluster, params):
 
 
 # pylint: disable=too-many-locals
-def _estimate_evcache_cluster_zonal(  # noqa: C901
+def _estimate_evcache_cluster_zonal(  # noqa: C901,E501 pylint: disable=too-many-positional-arguments
     instance: Instance,
     drive: Drive,
     desires: CapacityDesires,

service_capacity_modeling/models/org/netflix/kafka.py

@@ -151,7 +151,7 @@ def _kafka_read_io(rps, io_size_kib, size_gib, recovery_seconds: int) -> float:
 
 # pylint: disable=too-many-locals
 # pylint: disable=too-many-return-statements
-def _estimate_kafka_cluster_zonal(
+def _estimate_kafka_cluster_zonal(  # pylint: disable=too-many-positional-arguments
     instance: Instance,
     drive: Drive,
     desires: CapacityDesires,

service_capacity_modeling/models/org/netflix/rds.py

@@ -91,7 +91,7 @@ def _rds_required_disk_ios(disk_size_gib: int, db_type: str, btree_fan_out: int
     return math.log(pages, btree_fan_out)
 
 
-def _compute_rds_region(
+def _compute_rds_region(  # pylint: disable=too-many-positional-arguments
     instance: Instance,
     drive: Drive,
     needed_cores: int,

service_capacity_modeling/models/org/netflix/stateless_java.py

@@ -73,7 +73,7 @@ def _estimate_java_app_requirement(
     )
 
 
-def _estimate_java_app_region(
+def _estimate_java_app_region(  # pylint: disable=too-many-positional-arguments
     instance: Instance,
     drive: Drive,
     desires: CapacityDesires,

setup.py

@@ -9,7 +9,7 @@
     description="Contains utilities for modeling database capacity on a cloud",
     packages=setuptools.find_packages(exclude=("tests*", "notebooks*")),
     install_requires=[
-        "pydantic<2.0",
+        "pydantic>2.0",
         "scipy",
         "numpy",
         'importlib_resources; python_version < "3.7"',

tests/netflix/test_cassandra.py

@@ -5,8 +5,8 @@
 from service_capacity_modeling.interface import certain_float
 from service_capacity_modeling.interface import certain_int
 from service_capacity_modeling.interface import Consistency
-from service_capacity_modeling.interface import CurrentClusterCapacity
 from service_capacity_modeling.interface import CurrentClusters
+from service_capacity_modeling.interface import CurrentZoneClusterCapacity
 from service_capacity_modeling.interface import DataShape
 from service_capacity_modeling.interface import FixedInterval
 from service_capacity_modeling.interface import GlobalConsistency
@@ -311,7 +311,7 @@ def test_plan_certain():
     Use cpu utilization to determine instance types directly as supposed to
     extrapolating it from the Data Shape
     """
-    cluster_capacity = CurrentClusterCapacity(
+    cluster_capacity = CurrentZoneClusterCapacity(
         cluster_instance_name="i4i.8xlarge",
         cluster_instance_count=Interval(low=8, mid=8, high=8, confidence=1),
         cpu_utilization=Interval(

tox.ini

@@ -26,10 +26,11 @@ skip_install = true
 deps =
     pre-commit
     pylint
-    pydantic<2.0
+    pydantic>2.0
     scipy
     numpy
     isodate
+    setuptools
 commands =
     pre-commit run --all-files