Fix EITC calibration (#98)

* Add dropout

* Add dropout and EITC calibration

* Format

* Add install catch

* Change download folder

* Flip order of US install and bump US

* Add EITC targets

* Update data links

* Fix EITC returns are incorrectly calibrated #97

* Lower dropout rate to 5%

* Update validation

* Update data releases

changelog_entry.yaml

@@ -0,0 +1,4 @@
+- bump: minor
+ changes:
+ fixed:
+ - EITC calibration.

policyengine_us_data/datasets/acs/acs.py

@@ -111,7 +111,7 @@ class ACS_2022(ACS):
  time_period = 2022
  file_path = STORAGE_FOLDER / "acs_2022.h5"
  census_acs = CensusACS_2022
- url = "release://PolicyEngine/policyengine-us-data/1.9.0/acs_2022.h5"
+ url = "release://PolicyEngine/policyengine-us-data/1.10.0/acs_2022.h5"
 
 
 if __name__ == "__main__":

policyengine_us_data/datasets/cps/cps.py

@@ -54,6 +54,8 @@ def generate(self):
  raw_data.close()
  self.save_dataset(cps)
 
+ add_takeup(self)
+
 
 def add_rent(self, cps: h5py.File, person: DataFrame, household: DataFrame):
  cps["tenure_type"] = household.H_TENURE.map(
@@ -114,6 +116,30 @@ def add_rent(self, cps: h5py.File, person: DataFrame, household: DataFrame):
  cps["real_estate_taxes"][mask] = imputed_values["real_estate_taxes"]
 
 
+def add_takeup(self):
+ data = self.load_dataset()
+
+ from policyengine_us import system, Microsimulation
+
+ baseline = Microsimulation(dataset=self)
+ parameters = baseline.tax_benefit_system.parameters(self.time_period)
+ generator = np.random.default_rng(seed=100)
+
+ snap_takeup_rate = parameters.gov.usda.snap.takeup_rate
+ data["takes_up_snap_if_eligible"] = (
+ generator.random(len(data["spm_unit_id"])) < snap_takeup_rate
+ )
+
+ eitc_takeup_rates = parameters.gov.irs.credits.eitc.takeup
+ eitc_child_count = baseline.calculate("eitc_child_count").values
+ eitc_takeup_rate = eitc_takeup_rates.calc(eitc_child_count)
+ data["takes_up_eitc"] = (
+ generator.random(len(data["tax_unit_id"])) < eitc_takeup_rate
+ )
+
+ self.save_dataset(data)
+
+
 def uprate_cps_data(data, from_period, to_period):
  uprating = create_policyengine_uprating_factors_table()
  for variable in uprating.index.unique():
@@ -622,7 +648,7 @@ class CPS_2024(CPS):
  label = "CPS 2024 (2022-based)"
  file_path = STORAGE_FOLDER / "cps_2024.h5"
  time_period = 2024
- url = "release://policyengine/policyengine-us-data/1.9.0/cps_2024.h5"
+ url = "release://policyengine/policyengine-us-data/1.10.0/cps_2024.h5"
 
 
 class PooledCPS(Dataset):
@@ -681,7 +707,7 @@ class Pooled_3_Year_CPS_2023(PooledCPS):
  CPS_2023,
  ]
  time_period = 2023
- url = "release://PolicyEngine/policyengine-us-data/1.9.0/pooled_3_year_cps_2023.h5"
+ url = "release://PolicyEngine/policyengine-us-data/1.10.0/pooled_3_year_cps_2023.h5"
 
 
 if __name__ == "__main__":

policyengine_us_data/datasets/cps/enhanced_cps.py

@@ -25,7 +25,7 @@ def reweight(
  original_weights,
  loss_matrix,
  targets_array,
- dropout_rate=0.1,
+ dropout_rate=0.05,
 ):
  target_names = np.array(loss_matrix.columns)
  loss_matrix = torch.tensor(loss_matrix.values, dtype=torch.float32)
@@ -189,7 +189,7 @@ class EnhancedCPS_2024(EnhancedCPS):
  name = "enhanced_cps_2024"
  label = "Enhanced CPS 2024"
  file_path = STORAGE_FOLDER / "enhanced_cps_2024.h5"
- url = "release://policyengine/policyengine-us-data/1.9.0/enhanced_cps_2024.h5"
+ url = "release://policyengine/policyengine-us-data/1.10.0/enhanced_cps_2024.h5"
 
 
 if __name__ == "__main__":

policyengine_us_data/utils/loss.py

@@ -208,8 +208,16 @@ def build_loss_matrix(dataset: type, time_period):
  )
  eitc_eligible_children = sim.calculate("eitc_child_count").values
  eitc = sim.calculate("eitc").values
+ if row["count_children"] < 2:
+ meets_child_criteria = (
+ eitc_eligible_children == row["count_children"]
+ )
+ else:
+ meets_child_criteria = (
+ eitc_eligible_children >= row["count_children"]
+ )
  loss_matrix[returns_label] = sim.map_result(
- (eitc > 0) * (eitc_eligible_children == row["count_children"]),
+ (eitc > 0) * meets_child_criteria,
  "tax_unit",
  "household",
  )
@@ -219,7 +227,7 @@ def build_loss_matrix(dataset: type, time_period):
  f"irs/eitc/spending/count_children_{row['count_children']}"
  )
  loss_matrix[spending_label] = sim.map_result(
- eitc * (eitc_eligible_children == row["count_children"]),
+ eitc * meets_child_criteria,
  "tax_unit",
  "household",
  )