#!/usr/bin/env python
# utility_modules
from utility_modules.math_functions import expand_grid as expand_grid
from utility_modules.get_parameters import get_params
from utility_modules.get_parameters import get_parse_params
from utility_modules.import_zip import unzip_to_df as unzip_to_df
from utility_modules.output import Output as Output
from utility_modules.firm_calibration import Firmcal as Firmcal
# policy modules
from policy_modules.mcaid_elig import McaidEligibility as McaidEligibility
from policy_modules.exchange import Exchange as Exchange
from policy_modules.uninsured_penalty import UninsuredPenalty as UninsuredPenalty
from policy_modules.fpl import Fpl as Fpl
from policy_modules.premium_parameters import PremiumParams as PremiumParams
from policy_modules.firm_tax import FirmTax as FirmTax
from policy_modules.excise_tax import ExciseTax as ExciseTax
from policy_modules.employer_penalty import EmployerPenalty
from policy_modules.min_wage import MinWage as MinWage
from policy_modules.fam_tax import Tax as Tax
from policy_modules.fam_tax import StateTax as StateTax
from policy_modules.fam_tax import FedTax as FedTax
from policy_modules.cost_sharing_subsidy import CostSharing as CostSharing
from policy_modules.premium_calculate import CalculatePremium as CalculatePremium
from src.FamilyTable import Family as Family
from src.HieuTable import Hieu as Hieu
from src.WorkerTable import Worker as Worker
from src.FirmTable import Firm as Firm
from src.FirmChoice import FirmChoice
#python libraries
import datetime
import subprocess
import random
import time
import logging
import os
import re
from itertools import chain
from copy import deepcopy
import hashlib
import inspect
import shutil
# python libraries
from pandas.util.testing import assert_frame_equal
import scipy.optimize
import pandas as pd
import numpy as np
import math
import time
import pdb
[docs]class CalSim:
def __init__(self, path_dict = None, policy_dict = None, params_dict = None, cal_levers_dict = None, policy_levers_dict = None, system_sheet = None, general_dict = None):
"""
overarching class for the different tables in order to support better integration;
class arguments are to be passed as file locations
"""
print("initializing CalSim simulation")
self.logger = logging.getLogger('main.CalSim')
self.logger.debug('initiating CalSim simulation instance')
if (path_dict == None) | (policy_dict == None) | (params_dict == None) | (cal_levers_dict == None) | (policy_levers_dict == None) | (system_sheet == None) | (general_dict == None):
return
else:
# path information
Hieu_path = path_dict['Hieu_path']
Family_path = path_dict['Family_path']
Worker_path = path_dict['Worker_path']
Firm_path = path_dict['Firm_path']
Esi_path = path_dict['Esi_path']
C_params_path = cal_levers_dict['calibration_file']
self.calib_year=cal_levers_dict['calib_year']
self.current_path = path_dict['Current_path']
self.choice_path = path_dict['Choice_Tables_path']
self.output = path_dict['Out_path']
# read in tables
self.logger.debug('reading in input tables')
self.Hieu = Hieu(pd.read_csv(Hieu_path,memory_map=True))
hieu_df = self.Hieu.get_hieu_table()
hieu_df.dropna(subset=['person_id'], inplace=True)
hieu_df.reset_index(drop=True,inplace=True)
# create variables firm_id_of_exante_pholder and full_time_of exante_pholder
pholder_and_firm = hieu_df[["person_id","firm_id"]] #extract firm id
pholder_and_firm.columns = ["pholder_id_esi","firm_id_of_exante_pholder"]
full_time = hieu_df[["person_id","full_time"]] #extract full time indicator
full_time.columns = ["pholder_id_esi","full_time_of_exante_pholder"]
#Add this policy holder's firm id and full time status to hieu_df
#In self.hieu_table_paired_down:
#If firm_id_of_exante_pholder/full_time_of_exante_pholder is NA, the this person is not on ESI(no ESI policy holder).
self.hieu_table_paired_down = pd.merge(hieu_df, pholder_and_firm, on="pholder_id_esi", how="left")
self.hieu_table_paired_down = pd.merge(self.hieu_table_paired_down, full_time, on="pholder_id_esi", how="left")
# creating a balanced fake hieu for pivoting table purpose
self.Hieu.set_hieu_table(hieu_df)
self.Family0 = Family(pd.read_csv(Family_path,memory_map=True))
self.Family0['orig_fam_income'] = self.Family0['fam_income']
# replace negative income with zero income
self.Family0.loc[self.Family0['fam_income'] < 0, 'fam_income'] = 0
self.Family0.drop_duplicates('family_id',inplace=True)
self.Family0.reset_index(drop=True, inplace=True)
self.pt_hrs_esi_elig = int(general_dict['pt_hrs_esi_elig'])
self.Worker0 = Worker(pd.read_csv(Worker_path,memory_map=True))
self.Worker0['esi_not_elig'] = self.Worker0.hrs_week <=self.pt_hrs_esi_elig
self.Worker0['choice_default'] = 0 # adding a fake column for future choice
zero_wage_idx = np.where(self.Worker0['hourly_wage']==0)[0]
self.Worker0['hourly_wage'][zero_wage_idx]=7 # change workers who have an hourl wage of 0 to 7 (min wage)
seven_wage_idx = np.where(self.Worker0['hourly_wage']==7)[0]
new_wage = np.multiply(self.Worker0['hourly_wage'][seven_wage_idx], \
self.Worker0['hrs_week'][seven_wage_idx])*52
self.Worker0['wages'][seven_wage_idx] = np.maximum(new_wage, self.Worker0['wages'][seven_wage_idx])
self.Firm = Firm(pd.read_csv(Firm_path,memory_map=True))
# firm table shouldn't have duplicates, but..., data creation process is not perfect yet
self.Firm = self.Firm.drop_duplicates(['firm_id'])
# getting the list of bypass_firms, defined by government_firm ==1
self.bypass_firms = self.Firm['firm_id'][self.Firm.government_firm==1]
self.bypass_firms = self.bypass_firms.values.tolist()
# read in additional information
self.logger.debug('reading in static tables')
# partial choice indicator
self.partial_firm_choice = general_dict['partial_firm_choice']
self.use_pred_mcaid = general_dict['use_pred_mcaid']
# ken's method indicator
self.use_ken_flag=general_dict['ken']
self.Esi0 = pd.read_csv(Esi_path,memory_map=True) # for historical premium information
if self.partial_firm_choice:
index = np.where((self.Esi0.offer == 0) & (self.Esi0.choice != 0))[0]
self.Esi0=self.Esi0.drop(index).reset_index()
if self.use_ken_flag:
Ken_path=general_dict['kenadjusterpath']
self.ken_adjuster=pd.read_csv(Ken_path,memory_map=True,usecols=["person_id", "weight_adjuster"])
else:
self.ken_adjuster=None
# teleporting tables, anything that has dollar value
self.teleport_tables = {
"Worker" : "Worker0",
"Family" : "Family0",
"Esi" : "Esi0",
}
self.persons = ['adult_1', 'adult_2', 'child_1', 'child_2', 'adult_child_1', 'adult_child_2', 'adult_child_3', 'adult_child_4']
try:
# pull in relevant git commit id, branch, message
self.git_info = {}
self.git_info['calsim'] = (subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD']), \
subprocess.check_output(['git', 'rev-parse', 'HEAD']))
get_pol_input_commit_cmds = []
get_pol_input_commit_cmds.append('git --git-dir ' + general_dict['pol_input_path'] + '/.git --work-tree ' + \
general_dict['pol_input_path'] + ' rev-parse HEAD')
get_pol_input_commit_cmds.append('git --git-dir ' + general_dict['pol_input_path'] + '/.git --work-tree ' + \
general_dict['pol_input_path'] + ' log -1 --pretty=%B')
for cmd in get_pol_input_commit_cmds:
process = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE)
process.wait()
if process.returncode != 0:
print('Policy-input path does not have a git repository initialized.\nHas the folder been pulled from the Bitbucket repository?')
exit()
self.git_info['policy-input'] = (subprocess.check_output(get_pol_input_commit_cmds[0].split(' ')), \
subprocess.check_output(get_pol_input_commit_cmds[1].split(' '))[:-2])
# read in simulation parameters
self.logger.debug('reading in simulation parameters')
self.system_parameter = system_sheet
self.policy_parameter = policy_levers_dict
self.cal_parameter = cal_levers_dict
self.firm_parameters = get_parse_params(policy_dict['firm_behavior']) #delete policy_dict['fid']
self.inflate = get_params(policy_dict['inflators'])
self.pop0 = get_params(policy_dict['regional_pop_allages_17reg'])
self.pop1 = get_params(policy_dict['regional_pop_under65_17reg'])
# individual behavior tuning
self.firm_default = self.cal_parameter.firm_default
if self.firm_default:
self.calibrateRun = 1
self.use_cal_flag = True
else:
self.calibrateRun = int(self.cal_parameter.calibration)
self.use_cal_flag = self.cal_parameter.use_cal_flag
# long long table for debug
self.long_debug_run = general_dict['long_debug']
# get multipliers for calibration
self.cal_multipliers = self.cal_parameter.cal_multipliers
# utility parameters
self.utility_parameter = [self.cal_parameter.risk, self.cal_parameter.alpha]
# read in params_dict
self.debug = params_dict['debug']
self.track = params_dict['track']
# Process firm latent input
self.firm_latents_path = self.cal_parameter.firm_latents
self.firm_latent_calibration = self.cal_parameter.firm_latent_calibration
if self.firm_latent_calibration==True:
self.firm_threshold_to_maintain_offer = self.cal_parameter.firm_threshold_to_maintain_offer
self.firm_threshold_to_maintain_nooffer = self.cal_parameter.firm_threshold_to_maintain_nooffer
self.firm_threshold_to_maintain_offerPT = self.cal_parameter.firm_threshold_to_maintain_offerPT
if self.firm_latents_path is not None:
self.firm_latents = pd.read_csv(self.firm_latents_path, memory_map=True)
self.calibrate_ex_ante_latents =self.cal_parameter.calibrate_ex_ante_latents
if self.calibrate_ex_ante_latents ==True:
self.calibrate_ex_ante_variables={
'delta' :self.cal_parameter.delta,
'seed' :self.cal_parameter.seed,
'IM_range_under400' :self.cal_parameter.IM_range_under400,
'IM_range_over400' :self.cal_parameter.IM_range_over400,
'unins_delta' :self.cal_parameter.unins_delta,
'unins_random' :self.cal_parameter.unins_random,
'extra_esi_delta' :self.cal_parameter.extra_esi_delta,
'extra_esi_latent' :self.cal_parameter.extra_esi_latent,
'esi_crowdout_delta' :self.cal_parameter.esi_crowdout_delta,
'extra_medicaid_delta' :self.cal_parameter.extra_medicaid_delta,
}
# individual behavior parameters to be passed to HieuTable
self.utility_params = self.cal_parameter.utility_params
self.exp_multiplier = self.cal_parameter.exp_multiplier
self.flat_penalties = self.cal_parameter.flat_penalties
self.exp_xc_multiplier = self.cal_parameter.exp_xc_multiplier
self.oop_xc_multiplier = self.cal_parameter.oop_xc_multiplier
self.dual = self.cal_parameter.dual
self.mcaid_shadow_premium = self.cal_parameter.mcaid_shadow_premium
self.ab_choice = self.cal_parameter.ab_choice
self.exp_type = self.cal_parameter.exp_type # updating expenditure/oop variables
self.update_lv = self.cal_parameter.update_lv # file name for updating latent variable
self.exp_fname = self.cal_parameter.exp_fname # file name for updating expenditure variables
self.inc_adjuster = self.cal_parameter.inc_adjuster # income adjuster, pass it to individual behavior
self.bronze_ratio = self.cal_parameter.bronze_ratio # multiplier for bronze [0] for subsidized and [1] for unsubsidized
self.affordability_type = self.policy_parameter.affordability_definition #parameter for affordability
# pass policy_option to exchange module
self.policy_num = self.cal_parameter.policy_input_option
self.individual_params = {
'utility_params' : self.utility_params,
'exp_multiplier' : self.exp_multiplier,
'flat_penalties' : self.flat_penalties,
'dual' : self.dual,
'debug' : self.debug,
'mcaid_shadow_premium' : self.mcaid_shadow_premium,
'use_cal_flag' : self.use_cal_flag,
'ab_choice' : self.ab_choice,
'calib_flag' : self.calibrateRun,
'cal_multipliers' : self.cal_multipliers,
'utility_parameter' : self.utility_parameter,
'long_debug_run' : self.long_debug_run,
'bronze_ratio' : self.bronze_ratio,
'exp_xc_multiplier' : self.exp_xc_multiplier,
'oop_xc_multiplier' : self.oop_xc_multiplier,
'affordability_type' : self.policy_parameter.affordability_definition, #parameter for affordability
'use_pred_mcaid' : self.use_pred_mcaid
}
self.Latent_vars =None
if self.update_lv is not None: #Read in Latent Variables from the corresponding input
self.Latent_vars_path = os.path.join(self.current_path, self.update_lv)
self.Latent_vars = pd.read_csv(self.Latent_vars_path,memory_map=True)
self.afford_threshold = get_params(policy_dict['sub_parameters'])
#self.av_threshold = float(self.afford_threshold.Minimum_Value_Thresh)
self.esi_contr_cap = float(self.policy_parameter.esi_contr_cap)
# exchange related parameters, av, etc.
self.xc_admin_loading = get_params(policy_dict['xc_admin_loading'])
self.xc_admin_loading.set_index('type', inplace=True)
self.age_rating_parameters = get_params(policy_dict['age_rating'])
self.region_parameters = get_params(policy_dict['reg_xc_prem_17reg'])
# convert monthly regional exchange premiums to annual
for index, column in enumerate(self.region_parameters):
# for all column names != ['plan_year','region']
if index >= 2:
self.region_parameters[column] = self.region_parameters[column] * 12
# set year parameters
self.min_year = int(self.system_parameter.sim_year)
self.max_year = int(self.system_parameter.max_year)
self.base_year=int(self.system_parameter.base_year)
self.year_lst = self.system_parameter.years_to_run if 'years_to_run' in self.system_parameter.keys() else None
# firm decision threshold
self.offering_threshold = float(self.firm_parameters.Particip_needed_for_small_group)
# firm wage pass through parameters
self.passthru_dropcov_inc = float(self.system_parameter.passthru_dropcov_inc)
self.passthru_dropcov_dec = float(self.system_parameter.passthru_dropcov_dec)
self.passthru_addcov_inc = float(self.system_parameter.passthru_addcov_inc)
self.passthru_addcov_dec = float(self.system_parameter.passthru_addcov_dec)
# firm penalty amount multiplier
self.fplty_mult = float(self.system_parameter.fplty_mult)
# getting the threshold for predicting previously eligible
self.mcaid_exante_threshold = float(self.system_parameter.mcaid_exante_threshold)
self.firm_threshold = self.policy_parameter.firm_threshold
# individual mandate flag
self.individual_mandate = int(self.policy_parameter.individual_mandate)
# exchange premium inflator
self.xc_premium_inflator = float(self.policy_parameter.xc_premium_inflator)
# employer mandate flag
self.employer_mandate = int(self.policy_parameter.employer_penalty_policy)
# minimum wage flag
self.minWageFlag = int(self.policy_parameter.minimum_wage_policy)
self.minWage_base_year = int(self.system_parameter.base_year)
# shuffling firm indicator
self.firm_order = int(self.system_parameter.firm_order)
# set dynamic exchange premium parameters
self.dynamic_xc = int(self.policy_parameter.dynamic_xc_premium)
self.regional_spec = int(self.policy_parameter.regional_specificity)
self.dynamic_version = int(self.policy_parameter.dynamic_version)
# set flag for whether to allow medicaid people to choose other insurance types
self.medicaid_restriction = int(self.policy_parameter.medicaid_restriction)
# subsidy open to undoc flag
self.subsidy_to_undoc = int(self.policy_parameter.xc_subs_open_2_undoc)
# policy scenario: subsidy based on bronze for fpl> 400
self.bronze_subsidy = bool(self.policy_parameter.bronze_subsidy)
self.bronze_subsidy_year = int(self.policy_parameter.bronze_subsidy_year)
self.xc_premium_iteration = int(self.system_parameter.xc_premium_iteration)
self.xc_it = int(self.system_parameter.xc_it)
if (self.xc_it > 1):
self.xc_premium_iteration = self.xc_it
self.dynamic_xc = 1
print("Dynamic Exchange is on...")
self.xc_premium_method = int(self.system_parameter.xc_premium_method)
try:
self.xc_premium_tolerance = float(self.system_parameter.xc_premium_tolerance)
except:
self.xc_premium_tolerance = None
# logging information
self.logger.debug('starting simulation')
self.logger.debug('setting years ' + str(self.min_year) + ' to ' + str(self.max_year))
# running as test, or on entire firm table
self.test = general_dict['test']
self.percent = general_dict['percent']
self.config_path = general_dict['config_path']
self.firm_list = map(int, general_dict['firm_list'].split(','))
self.pmc_flag = general_dict['pmc_flag']
# choice file generation
self.force_inf_choice = general_dict['force_inf_choice']
# Generate function parameters for each module
except KeyError as e:
self.logger.critical('could not initialize system parameters: %s' % e)
raise KeyError('could not initialize system parameters: %s' % e)
# read in the c params table
if self.use_cal_flag == 1:
self.C_params = pd.read_csv(C_params_path,memory_map=True)
#read in policy modules
try:
print('setting up policy modules')
self.logger.debug('reading in policy modules')
print('setting up mcaid')
self.Mcaid = McaidEligibility(policy_dict, self.system_parameter)
print('setting up fpl')
self.Fpl = Fpl(policy_dict, self.system_parameter)
print('setting up prem')
self.Prem = CalculatePremium(policy_dict, self.system_parameter['sim_year'])
print('setting up exchange')
self.Exchange = Exchange(policy_dict)
print('setting up uninsured penalty')
self.UninsuredPenalty = UninsuredPenalty(policy_dict)
print('setting up premium params')
self.PremiumParams = PremiumParams(policy_dict)
print('setting up firm tax')
self.FirmTax = FirmTax(policy_dict)
print('setting up employer penalty')
self.EmployerPenalty = EmployerPenalty(policy_dict)
print('setting up excise tax')
self.ExciseTax = ExciseTax(policy_dict)
print('setting up min wage')
self.MinWage = MinWage(policy_dict, self.min_year, self.minWage_base_year)
print('setting up fed tax')
self.FedTax = FedTax(policy_dict['fed_tax_rates'], policy_dict['fed_std_deduction'])
print('setting up state tax')
self.StateTax = StateTax(policy_dict['state_tax_rates'], policy_dict['state_std_deduction'])
print('setting up cost sharing')
self.CostSharing = CostSharing(policy_dict)
#self.Hieu.set_utility(self.individual_params) # moved to set_year
except KeyError:
self.logger.critical('could not initialize policy module')
raise KeyError
self.inflate_tables = {
'Worker' : ['hourly_wage', 'wages'],
'Family' : ['fam_agi', 'fam_income'],
'Esi' : ['premium_self_actual', 'premium_sp_actual', 'premium_ch_actual', 'premium_fam_actual',
'oop_self_actual', 'oop_sp_actual', 'oop_ch_actual', 'oop_fam_actual'],
'Hieu' : ['ind_income']
}
self.inflate_choices = ['exp' + str(i) for i in range(8)] + ['oop' + str(i) for i in range(8)] + \
['var' + str(i) for i in range(8)] + \
['exp' + str(i) + '_adj' for i in range(8)] + \
['oop' + str(i) + '_adj' for i in range(8)]
self.inflate_latent_variables = ['cal' + str(i) for i in range(8)] + ['new_cal' + str(i) for i in range(8)]
medical_cost_list = ['latent_vars', 'flat_penalties', 'oop_self_actual', 'oop_sp_actual', 'oop_ch_actual', 'oop_fam_actual'] + \
self.inflate_choices + self.inflate_latent_variables
cpiu_list = ['hourly_wage', 'wages', 'minwage_threshold', 'ind_income', 'fam_income', 'fam_agi']
ESI_premium_list = ['premium_self_actual', 'premium_sp_actual', 'premium_ch_actual', 'premium_fam_actual']
self.inflate_map = dict({i : 'Medical_Cost' for i in medical_cost_list}, **{i : 'cpiu' for i in cpiu_list})
self.inflate_map.update({i : 'ESI_premium' for i in ESI_premium_list})
# initialize unique tables that require a unique shallow handling
self._uniq_tables = ['Worker', 'Family'] # these are choice-specific tables during the simulation process
# develop ESI_0 table from choice 0
self.Esi_0 = self.Esi0[self.Esi0['choice'] == 0]
# initialize table structure and policy dictionary
self._build_teleport()
# number of iteration of esi premium loop
self.prem_loop_iter = int(self.system_parameter.premium_iterations)
# comparing the firm threshold defined on firm behavior tab with from the config file
if self.system_parameter.savings is not None:
self.firm_threshold_continue_offering = float(self.system_parameter.savings[0])
self.firm_threshold_continue_not_offering = float(self.system_parameter.savings[1])
else:
self.firm_threshold_continue_offering = float(self.firm_threshold[0])
self.firm_threshold_continue_not_offering = float(self.firm_threshold[1])
# initialize output class
self.output = Output(self.output, self.git_info, general_dict, params_dict, cal_levers_dict, policy_levers_dict, system_sheet, self.regional_spec)
# optimization method
self.optimization_method = int(general_dict['optimization_method'])
# for calibration purpose, we only run the default choice ### also turn off use_cal_flag
if self.calibrateRun is 1 and self.cal_parameter.firm_default is False:
self.use_cal_flag = False
if self.calibrateRun:
self.Esi0 = self.Esi0.loc[self.Esi0.choice == 0] if self.long_debug_run == 0 else self.Esi0
[docs] def start_simulation(self):
"""
high level method hook to start simulation
"""
try:
random.seed(self.system_parameter.seed)
except AttributeError:
self.logger.critical('could not find system attributes... quitting simulation.')
return
# check year are valid or raise exception
if self.min_year <= self.max_year:
year_range = range(self.min_year, self.max_year + 1)
else:
raise ValueError('year range of ' + str(min_year) +
'to ' + max_year + 'is not valid')
# check if a year list has been passed in
if self.year_lst is not None:
year_range = self.year_lst
WorkerOrig = self.Worker.copy(True)
FamOrig = self.Family.copy(True)
HieuOrig = self.Hieu.get_hieu_table().copy(True)
for y in year_range:
print('setting year: ' + str(y))
self._set_year(y)
WorkerInit = self.Worker.copy(True) #used to pass the firewall between choices
FamInit = self.Family.copy(True) #used to pass the firewall between choices
print("Year = ", y)
self.simulate_year(y, WorkerInit, FamInit, self.ab_choice)
self.Worker = WorkerOrig.copy(True)
self.Family = FamOrig.copy(True)
self.Hieu.set_hieu_table(HieuOrig.copy(True))
self.cleanup()
def _set_year(self, year):
"""
set years and transports policy modules
"""
# logging information
self.logger.debug('setting year: ' + str(year))
if self.debug:
print("setting up for year: " + str(year))
# set year parameters
self.curr_year = year
self.Mcaid.set_year(year)
self.Fpl.set_year(year)
if (self.employer_mandate):
self.EmployerPenalty.set_year(year)
self.afford_threshold = self.afford_threshold[self.afford_threshold['Year']==year]
self.av_threshold = float(self.afford_threshold.Minimum_Value_Thresh)
# Exchange module
self.exchange_params = {
'xc_inflator': self.xc_premium_inflator,
'av_threshold': self.av_threshold,
'pmc_flag': self.pmc_flag,
'bronze_subsidy': self.bronze_subsidy,
'bronze_subsidy_year': self.bronze_subsidy_year,
'policy_num': self.policy_num
}
# extract relevant year data and re-index from 0
if self.regional_spec == 1:
curr_rp = (self.region_parameters.loc[self.region_parameters['plan_year']==year]).reset_index(drop=True)
curr_rp = curr_rp.drop(['plan_year'], axis=1)
self.Exchange.set_year(year, self.subsidy_to_undoc, self.afford_threshold, self.exchange_params, curr_rp)
else:
self.Exchange.set_year(year, self.subsidy_to_undoc, self.afford_threshold, self.exchange_params)
self.ExciseTax.set_year(year)
self.FedTax.set_year(year)
self.CostSharing.set_year(year)
self.UninsuredPenalty.set_year(year)
self.MinWage.set_year(year)
self.output.set_year(year)
self._inflate_table_year(year)
self.StateTax.set_year(year)
## flat penalty amount is inflated to future year
if (self.curr_year > self.calib_year):
year_idx = (self.inflate.year > self.calib_year) & (self.inflate.year<=self.curr_year)
inflator = np.prod(self.inflate[self.inflate_map['flat_penalties']][year_idx])
self.flat_penalties = [x * inflator for x in self.flat_penalties]
# adjust for population growth
if self.debug is True:
print('population pre-inflation: ' + str(self.Hieu.get_hieu_table()['p_weight'].sum()))
if 'region' in self.Hieu.get_hieu_table() and self.regional_spec == 1:
self.grow_reg_pop(year)
else:
self.grow_statewide_pop(year)
if self.debug is True:
print('population post-inflation: ' + str(self.Hieu.get_hieu_table()['p_weight'].sum()))
hieu_df = self.Hieu.get_hieu_table()
self.hieu_df_info = hieu_df[['u_esi', 'u_priv', 'u_unins', 'u_mcaid', 'u_mcare', 'c_esi', 'c_priv', 'c_mcaid', 'c_mcare', 'c_unins']]
hieu_df.drop( ['u_esi', 'u_priv', 'u_unins', 'u_mcaid', 'u_mcare', 'c_esi', 'c_priv', 'c_mcaid', 'c_mcare', 'c_unins'],axis=1,inplace=True)
if 'fpl' in hieu_df:
hieu_df = hieu_df.drop(['fpl','fam_income','family_size'], axis=1)
hieu_df['ind_fpl']= self.Fpl.calculate_ind_fpl(hieu_df)
self.Family['fpl'] = self.Fpl.calculate_fpl(self.Family) # calculate fpl
pop_df = self.propagate_fpl_changes(hieu_df, self.Family, self.Worker, self.Esi_0,self.affordability_type)
hieu_df.update(pop_df)
self.Hieu.set_hieu_table(hieu_df)
self.Hieu.set_pop_hieu(pop_df)
dict2={'flat_penalties': self.flat_penalties}
self.individual_params.update(dict2)
self.Hieu.set_utility(self.individual_params)
# expenditure and ooop for output
self.Expenditures = pd.DataFrame()
# minimum wage policy is applied
if (self.minWageFlag):
(self.Worker, self.Family,self.Hieu) = self.MinWage.min_wage(self.Worker, self.Family,self.Hieu) # updating wage and income
self.Family['fpl'] = self.Fpl.calculate_fpl(self.Family) # updating fpl
hieu_df = self.Hieu.get_hieu_table()
hieu_df['ind_fpl'] = self.Fpl.calculate_ind_fpl(hieu_df)
# update hieu table medicaid eligiblity
pop_df = self.propagate_fpl_changes(hieu_df, self.Family, self.Worker, self.Esi_0,self.affordability_type)
# calculate new previously eligible column adjusted to simulated year's fpl changes
pop_df['adjusted_elig_exante'] = self.Mcaid.produce_elig_exante(pop_df)
unelig_mcaid_idx = np.where((pop_df['mcaid_elig_exante']==1) & (pop_df['gvmt_insurance']==1) & ( pop_df['adjusted_elig_exante']==0))
pop_df['mcaid_elig_flipped']= (pop_df['mcaid_elig_exante']==1) &( pop_df['adjusted_elig_exante']==0)
pop_df['mcaid_elig_exante'] = pop_df['adjusted_elig_exante']
pop_df.drop('adjusted_elig_exante',axis=1)
pop_df['gvmt_insurance'][unelig_mcaid_idx[0]] = 0 # getting those off medicaid
hieu_df['mcaid_elig_flipped']=pop_df.mcaid_elig_flipped
hieu_df.update(pop_df)
self.Hieu.set_hieu_table(hieu_df)
self.Hieu.set_pop_hieu(pop_df)
[docs] def grow_reg_pop(self, year):
"""
inflates the population for the simulated year
self.pop0 describes population for all ages and self.pop1 for population of < 65
"""
# first extract person's weights information
hieu_df = self.Hieu.get_hieu_table()
weights = hieu_df[['person_id','age','region','p_weight']]
weights = weights[weights['person_id'].notnull()]
# split ages up into bins
bins = [0, 65, 150]
group_names = ['0 - 65', '65+']
weights['age_bins'] = pd.cut(weights['age'], bins, labels=group_names, right=False, include_lowest=True)
aggre = weights.groupby(['age_bins','region']).sum().reset_index().drop(['person_id','age'], axis=1)
# retrieve information from population table into df and merge original weights into it
# use pops from simulated year and pops from base year to calculate rate
under65_agg = aggre.loc[aggre.age_bins == '0 - 65']
under65_agg['growth_rate'] = (self.pop1['y'+str(year)].astype(float))/(under65_agg.p_weight.astype(float))
over65_agg = aggre.loc[aggre.age_bins == '65+'].reset_index(drop=True)
over65_agg['growth_rate'] = ((self.pop0['y'+str(year)]-self.pop1['y'+str(year)]).astype(float))/(over65_agg.p_weight.astype(float))
aggre = pd.concat([under65_agg, over65_agg])
# merge back into weights to calculate new rates
weights = pd.merge(weights, aggre[['age_bins','region','growth_rate']], on=['age_bins','region'], how='left')
weights['p_weight'] = weights['p_weight']*weights['growth_rate']
# merge back into Hieu table
temp = hieu_df.drop('p_weight', axis=1)
temp = pd.merge(temp, weights[['person_id','p_weight']], on='person_id', how='left')
hieu_df['p_weight'] = temp['p_weight']
self.Hieu.set_hieu_table(hieu_df)
[docs] def grow_statewide_pop(self, year):
# first extract person's weights information
hieu_df = self.Hieu.get_hieu_table()
weights = hieu_df[['person_id','age','p_weight']]
weights = weights[weights['person_id'].notnull()]
# split ages up into bins
bins = [0, 65, 150]
group_names = ['0 - 65', '65+']
weights['age_bins'] = pd.cut(weights['age'], bins, labels=group_names, right=False, include_lowest=True)
aggre = weights.groupby(['age_bins'], as_index=False).sum().drop(['person_id','age'], axis=1)
# retrieve information from population table into df and merge original weights into it
# use pops from simulated year and pops from base year to calculate rate
under65_agg = aggre.loc[aggre.age_bins == '0 - 65']
under65_agg['growth_rate'] = (self.pop1['y'+str(year)].sum().astype(float))/(under65_agg.p_weight[0].astype(float))
over65_agg = aggre.loc[aggre.age_bins == '65+'].reset_index(drop=True)
over65_agg['growth_rate'] = ((self.pop0['y'+str(year)].sum()-self.pop1['y'+str(year)].sum()).astype(float))/(over65_agg.p_weight.astype(float))
aggre = pd.concat([under65_agg, over65_agg])
# merge back into weights to calculate new rates
weights = pd.merge(weights, aggre[['age_bins','growth_rate']], on=['age_bins'], how='left')
weights['p_weight'] = weights['p_weight']*weights['growth_rate']
# merge back into Hieu table
temp = hieu_df.drop('p_weight', axis=1)
temp = pd.merge(temp, weights[['person_id','p_weight']], on='person_id', how='left')
hieu_df['p_weight'] = temp['p_weight']
self.Hieu.set_hieu_table(hieu_df)
[docs] def simulate_year(self, year, worker_table, fam_table, which_exp_to_use):
"""
simulation for the given year
passing a copy of Worker, and Family down stream
"""
WorkerOrig, FamOrig = worker_table.copy(True), fam_table.copy(True)
self.logger.debug("setup duration: %f" % time.clock())
self.logger.debug('starting simulation of year ' + str(year))
if self.test:
if self.percent:
f_lst = self.Firm.firm_id.unique()
random.shuffle(f_lst)
f_lst = f_lst[: int(len(f_lst)*self.percent)]
else:
f_lst = self.Firm.firm_id.unique()
# keep only fake firms
new_lst = []
firm_str = map(str, f_lst)
for index, i in enumerate(firm_str):
if i.startswith('55'):
new_lst.append(i)
f_lst = map(int, new_lst)
# previously offering firm
f_lst = self.firm_list
else:
if self.firm_order is 2:
print("descending firm size order")
firms = self.Firm.sort_values(['firm_size', 'firm_id'], ascending=False).firm_id.unique()
#firms = firms[700:725]
#firms = firms[730:750]
elif self.firm_order is 3:
print("ascending firm size order")
firms = self.Firm.sort_values(['firm_size', 'firm_id'], ascending=True).firm_id.unique()
elif self.firm_order is 4:
print("ascending firm id order")
firms = self.Firm.sort_values(['firm_size', 'firm_id'], ascending=True).firm_id.unique()
f_lst = firms.tolist()
f_lst = [int(x) for x in f_lst] # the order of the firms is predetermined by firm size
n_selected_firms = len(f_lst)
if self.firm_order is 1:
print("shuffling firms")
random.shuffle(f_lst)
### obtaining initial annual exchange premium
xc_silver_21 = float(self.Exchange.slv_prem)
xc_bronze_21 = float(self.Exchange.brz_prem)
xc_catast_21 = float(self.Exchange.cat_prem)
## creating a panda dataframe for future use, and this will be updated during exchange dynamic premium loop
xc_mat = [(6, xc_silver_21), (7,xc_bronze_21),(8,xc_catast_21)]
xc_prem = pd.DataFrame(xc_mat, columns=['type', 'prem'])
self.statewide_premiums=xc_prem.copy()
self.statewide_premiums["iteration"]=0
xc_prem = xc_prem.set_index('type')
self.person_count = 0
self.weight_total = 0
if self.use_cal_flag:
try:
self.C_params = self.C_params[['person_id','c_unins_2015','c_self_esi_2015','c_spouse_esi_2015','c_parent1_esi_2015',
'c_parent2_esi_2015','c_xc_silver_2015','c_xc_bronze_2015', 'c_xc_cat_2015','c_mcaid_2015','c_mcare_2015','mcaid_predict']]
except:
print("c param column headers not found, columns may not align properly")
#test_lst = []
### adding dynamic exchange premium loop
if self.dynamic_xc == 1:
self.xc_parameters = pd.concat([xc_prem,self.xc_admin_loading], axis=1)
print("Running Dynamic Exchange Premium")
#create a dataframe that is going to keep track of
#all regional premiums for all dynamic exchange iterations
#the initial data in this dataframe will be the regional
#premiums read from policy input with iteration=0
self.regional_premiums=self.Exchange.get_reg_prem().copy()
self.regional_premiums["iteration"]=0
for i in range(0, int(self.xc_premium_iteration+1)):
prev_xc = xc_prem.loc[6][0]
# reset input tables to original state
worker_table = self.Worker.copy(True)
fam_table = self.Family.copy(True)
print("xc_prem at iteration %d" % i)
print(xc_prem)
#this extra iteration is used for printing out xc_prem for the
#last iteration purpose
if i==int(self.xc_premium_iteration):
break
for index, f in enumerate(f_lst):
try:
print("working on firm: %d/%d: %d, on xc prem iteration %d" % (index+1, n_selected_firms, f, i+1))
self._setup_firm(f)
self.Expenditures = self.Expenditures.append(self.exp_firm, ignore_index= True)
self.Expenditures.drop_duplicates('person_id', inplace=True)
(updated_worker_table, updated_family_table) = self.premium_loop(xc_prem, worker_table, fam_table, \
self._prev_offer, self.individual_mandate)
(worker_table, fam_table) = (updated_worker_table, updated_family_table)
except IOError:
self.logger.critical('could not find choice file for ' + str(f))
### update exchange premium, for dynamic exchange premium
if 'region' in self.Hieu.get_hieu_table():
prem_ind_choices = self._wide_to_long(self.perm_ind_choices, ['p_id', 'xc_prem', 'exp', 'exp_adj', 'oop','oop_adj','xc_sub_','sub_amount_'], ['region'])
prem_ind_choices.rename(columns={'hieu_region': 'region'}, inplace=True)
else:
prem_ind_choices = self._wide_to_long(self.perm_ind_choices, ['p_id', 'xc_prem', 'exp', 'exp_adj', 'oop','oop_adj','xc_sub_','sub_amount_'])
prem_ind_choices = prem_ind_choices.loc[prem_ind_choices['choice'].isin([6,7,8])]
prem_ind_choices = pd.merge(prem_ind_choices, self.Hieu.get_pop_hieu()[['hieu_id', 'person_type', 'age', 'n_person','subsidy_amount_total']], on=(['hieu_id', 'person_type']))
# calculate xc premium based on method chosen 1 = statewide first then recaliberating regional 2 = by regional
# always assuming regional analysis [xc 2/23/2018]
if self.xc_premium_method == 1:
# updating state-wide premiums
#test_lst.append(prem_ind_choices)
xc_prem = self._cal_exchange_premiums(prem_ind_choices, which_exp_to_use)
xc_prem = pd.DataFrame(xc_prem.items(), columns=['type', 'prem'])
xc_prem['iteration']=i+1
self.statewide_premiums=pd.concat([self.statewide_premiums,xc_prem])
xc_prem = xc_prem.set_index('type')
# update the state-wide premiums and calculate regional premiums based on new state-wide premiums
self.regional_premiums=pd.concat([self.regional_premiums,self.Exchange.dynamic_exchange_update(xc_prem,i+1)])
if self.xc_premium_tolerance != None and abs(xc_prem.loc[6][0] - prev_xc) / prev_xc < self.xc_premium_tolerance:
break
prev_xc = xc_prem.loc[6][0]
else:
for index, f in enumerate(f_lst):
try:
if self.track==True:
print("working on firm: %d/%d: %d without dynamic exchange premium" % (index+1, n_selected_firms, f))
self._setup_firm(f)
self.Expenditures = self.Expenditures.append(self.exp_firm, ignore_index=True)
self.Expenditures.drop_duplicates('person_id', inplace=True)
(updated_worker_table, updated_family_table) = self.premium_loop(xc_prem, worker_table, \
fam_table, self._prev_offer, self.individual_mandate)
(worker_table, fam_table) = (updated_worker_table, updated_family_table)
# premium_loop calls firm comparisons
# after firm has chosen it choice, the self.Fam_Table and self.Worker_Table will be updated
# reflect te choice by current firm
except IOError:
print('IOError: could not find choice file for ' + str(f))
self.logger.critical('could not find choice file for ' + str(f))
exit()
(self.Worker, self.Family) = (worker_table, fam_table)
self.perm_ind_choices = self._adjust_duals(self.perm_ind_choices)
if 'indiv_fam_cap_default0' in self.perm_ind_choices:
to_melt = ['p_id', 'ins_p', 'firm_id', 'plan_id', 'var', 'xc_sub_','adultkid_xc_sub_','mcaid_', 'new_mcaid_', 'pred_mcaid_',
'penalty','penalty_orig','esi_premium_','wage_delta_', 'age_rated_premium_','xc_prem', 'xc_prem_un_sub','sub_amount_', 'sub_amount_default', 'bronze_only_', 'xc_diff_', 'cal',
'exp', 'exp_adj', 'oop_adj', 'oop', 'exemption_','indiv_fam_cap_','indiv_fam_cap_default', 'sub_chosen'] + (['dual_'] if self.dual else [])
hieu_specific_cols = (['region'] if 'region' in self.perm_ind_choices else []) + ['utility', 'oop', 'tot_c','subsidy_amount_total','subsidy_bronze_total','subsidy_amount_total_default']
else:
to_melt = ['p_id', 'ins_p', 'firm_id', 'plan_id', 'var', 'xc_sub_','adultkid_xc_sub_','mcaid_', 'new_mcaid_', 'pred_mcaid_',
'penalty','penalty_orig','esi_premium_', 'wage_delta_', 'age_rated_premium_', 'xc_prem', 'xc_prem_un_sub','sub_amount_', 'bronze_only_', 'xc_diff_', 'cal',
'exp', 'exp_adj', 'oop_adj', 'oop', 'exemption_','indiv_fam_cap_', 'sub_chosen'] + (['dual_'] if self.dual else [])
hieu_specific_cols = (['region'] if 'region' in self.perm_ind_choices else []) + ['utility', 'oop','tot_c','subsidy_bronze_total','subsidy_amount_total']
long_output = self._wide_to_long(self.perm_ind_choices, to_melt, hieu_specific_cols)
# save individual/choice specific data
if self.calibrateRun == 1:
if self.firm_default:
self.output.initiate_out_table(1, 'firm_default_utility_table.csv', self.long_calib_table)
else:
self.output.initiate_out_table(1, 'long_calibration_table.csv', self.long_calib_table)
if self.long_debug_run:
self.output.initiate_out_table(1, 'long_calibration_table_non_default.csv', self.long_calib_table_not_default)
# print debugging information to terminal
self.firm_choices_all = pd.merge(self.firm_choices_all, self.Firm[['firm_id', 'firm_wgt', 'fake_firm']], on='firm_id', how='left')
self.firm_choices_all = pd.merge(self.firm_choices_all, self.Esi[['firm_id', 'choice', 'pt','contr_p_self','contr_p_fam','contr_p_sp','contr_p_ch','oop_self_actual', 'oop_ch_actual',
'oop_sp_actual', 'oop_fam_actual']], on=['firm_id', 'choice'], how='left')
self.firm_choices_all['esi_self_cost']=self.firm_choices_all.contr_p_self*self.firm_choices_all.premium_self_actual
self.firm_choices_all['esi_fam_cost']=self.firm_choices_all.contr_p_fam*self.firm_choices_all.premium_fam_actual
self.firm_choices_all['esi_sp_cost'] = self.firm_choices_all.contr_p_sp * self.firm_choices_all.premium_sp_actual
self.firm_choices_all['esi_ch_cost'] = self.firm_choices_all.contr_p_ch * self.firm_choices_all.premium_ch_actual
#### updating everything after all firms are done
hieu_df = self.Hieu.get_hieu_table()
pop_df = self.Hieu.get_pop_hieu()
# merge the latest family_income
pop_df.drop(['fpl', 'fam_income', 'family_size'], axis=1, inplace=True)
pop_df = pd.merge(pop_df, self.Family[['family_id', 'fpl', 'fam_income', 'family_size']], on='family_id',how='left')
pop_df.drop_duplicates(inplace=True)
pop_df = self.propagate_fpl_changes(hieu_df, self.Family, self.Worker, self.Esi_0, self.affordability_type)
hieu_df.update(pop_df)
pop_df['esi_threshold']=float(self.afford_threshold.Esi_Affordable_Thresh)
# add esi_cov column to Hieu
offering_firms = self.firm_choices_all[(self.firm_choices_all['chosen_choice'] == 1) & \
(self.firm_choices_all['offer'] == 1)].merge(self.Worker, on='firm_id') \
[['firm_id', 'offer', 'pt', 'full_time', 'hieu_id', 'person_id']]
offering_firms = offering_firms.merge(pop_df[['person_id', 'person_type']])
covered_hieus = offering_firms[(((offering_firms['person_type'] == 0) | \
(offering_firms['person_type'] == 1)) == 1) & (((offering_firms['pt'] == 0) & \
(offering_firms['full_time'] == 0)) == 0)].drop_duplicates('hieu_id')
covered_hieus['esi_cov'] = 1
new_hieu_table = pop_df.merge(covered_hieus[['hieu_id', 'esi_cov']], how='left', on='hieu_id')
new_hieu_table.loc[:,'esi_cov'].fillna(0, inplace=True)
# for nonexistent people, set flag to NaN
new_hieu_table.loc[new_hieu_table['person_id'] != new_hieu_table['person_id'],'esi_cov'] = np.nan
# adult child case
covered_adult_child = offering_firms[(((offering_firms['person_type'] == 0) | (offering_firms['person_type'] == 1)) == 0) & (((offering_firms['pt'] == 0) & (offering_firms['full_time'] == 0)) == 0)]
covered_adult_child['esi_cov'] = 1
covered_adult_child = pop_df.merge(covered_adult_child[['person_id', 'esi_cov']], how='left', on='person_id')
new_hieu_table.loc[covered_adult_child['esi_cov'] == 1,'esi_cov'] = 1
hieu_df.update(pop_df.set_index(['hieu_id', 'person_type'], inplace=True))
hieu_df = pd.concat([hieu_df, self.hieu_df_info], axis=1)
new_hieu_table = pd.concat([new_hieu_table, self.hieu_df_info], axis=1)
self.Hieu.set_hieu_table(hieu_df)
self.Hieu.set_pop_hieu(new_hieu_table)
#firm choices
#choice: available choice for firms
#chosen_choice: indicator of whether firm picks that choice or not
#indicators for whether the firm drops ESI coverage entirely or only drops PT ESI coverage or not
self.firm_choices_all["firm_dropped_my_cov"]=True
self.firm_choices_all["firm_of_pholder_dropped_PTcov"]=True
#firms that previously offer ESI at least to FT
prev_offer_firms = self.firm_choices_all.loc[(self.firm_choices_all.chosen_choice==1)&(self.firm_choices_all.prev_offer==1),]
self.dropped_firms=prev_offer_firms.loc[prev_offer_firms.offer==0,["firm_id","firm_dropped_my_cov"]]
self.dropped_firms.columns=["firm_id_of_exante_pholder","firm_dropped_my_cov"]
#offer==True if at least FT covered
#exante offer FT+PT: choice==0 continue; choice==1 drop only PT; choice==2 drop both
#exante offer FT-PT: choice==0 continue; choice==1 drop both
#exante offer -FT-PT: choice==0 continue; choice==1 add coverage with typical AV and typical employee contribution;
# choice==2 add coverage with bronze AV and typical employee coverage
# choice==3 add coverage with cronze AV and high employee coverage
#check with Dave/Miranda with logic
self.dropped_PT_firms=prev_offer_firms.loc[((prev_offer_firms.choice==1)&(prev_offer_firms.offer==1)),["firm_id","firm_of_pholder_dropped_PTcov"]]
self.dropped_PT_firms.columns=["firm_id_of_exante_pholder","firm_of_pholder_dropped_PTcov"]
#merge back into hieu_table
#If firm_dropped_my_cov is NA, then the person previoulsy is not on ESI or the firm keeps FT/PT ESI.
#If firm_of_pholder_dropped_PTcov is NA, then either the person previously is not on ESI or the firm drops FT ESI.
self.hieu_table_paired_down=pd.merge(self.hieu_table_paired_down,self.dropped_firms,on="firm_id_of_exante_pholder",how="left")
self.hieu_table_paired_down=pd.merge(self.hieu_table_paired_down,self.dropped_PT_firms,on="firm_id_of_exante_pholder",how="left")
#check what are used from hieu_choice.csv
#can we directly take care of these issues before outputting or we have to wait for the output part necessarily?
if self.debug is True:
print("firm choice")
print(self.firm_choices_all)
if self.calibrateRun:
output_calib = self.long_calib_table
else:
output_calib = None
if self.firm_latent_calibration ==True:
self.Firmcal=Firmcal(self.firm_threshold_to_maintain_offer,self.firm_threshold_to_maintain_nooffer,self.firm_threshold_to_maintain_offerPT,self.firm_choices_all)
firm_latents=self.Firmcal.create_firm_latent()
else:
firm_latents=None
if firm_latents is not None:
self.output.initiate_out_table(1, 'firm_latents.csv', firm_latents)
# create all output tables
self.output.initialize_raw_data({
'perm_ind_choices' : self.perm_ind_choices,
'long_output' : long_output,
'long_calib_table' : output_calib,
'firm_choices_all' : self.firm_choices_all,
'hieu_table' : self.Hieu.get_hieu_table(),
'pop_hieu' : self.Hieu.get_pop_hieu(),
'firm_table' : self.Firm,
'fam_table' : self.Family,
'worker_table' : self.Worker,
'fam_table_orig' : FamOrig,
'worker_table_orig' : WorkerOrig,
'esi_table' : self.Esi,
'exp_table' : self.Expenditures,
'weights' : (self.person_count, self.weight_total),
'dual_parameter' : self.dual,
'use_ken_flag' : self.use_ken_flag,
'ken_adjuster' : self.ken_adjuster,
'dynamic_version' : self.dynamic_version,
'dynamic_exchange_statewide_premiums' : self.statewide_premiums,
'dynamic_exchange_regional_premiums' : self.regional_premiums,
'hieu_table_paired_down' : self.hieu_table_paired_down,
'dropped_firms' : self.dropped_firms,
'dropped_PT_firms' : self.dropped_PT_firms,
'calibrateRun' : self.calibrateRun,
'calibrate_ex_ante_latents' : self.calibrate_ex_ante_latents,
'Latent_vars' : self.Latent_vars
})
firmsize_threshold = self.EmployerPenalty.penalty_table.loc[self.EmployerPenalty.penalty_table['year']==year].iloc[0]['min_fte']
self.output.build_out_tables(firmsize_threshold)
self.output.output_tables()
self.output.write_config()
# write optimization depth
#print 'Max Depth Distribution (# iterations)'
#print pd.DataFrame(self.max_iters).describe()
def _adjust_duals(self, choices):
for i, person in enumerate(self.persons):
choices.loc[(choices[person] == 9) & (choices['mcaid_'+str(i)] == False), 'xc_sub_'+str(i)] = 0
choices.loc[(choices[person] == 9) & (choices['mcaid_'+str(i)] == False), 'mcaid_'+str(i)] = True
return choices
### set up for the current firm in focus;
def _setup_firm(self, f):
"""
sets up current firm
"""
self.logger.debug('setting firm ' + str(f))
self.curr_firm = f
self.Firm.curr_firm = f
# read in firm choice
self._read_choice()
choice_lst = self.Esi.choice[self.Esi.firm_id == self.curr_firm]
if self.debug is True:
ppl_count = self.Worker.loc[self.Worker.firm_id == self.curr_firm].person_id.tolist()
hieu_count = self.Hieu.get_hieu_table()[['person_id', 'hieu_id', 'p_weight']]
hieu_match = hieu_count.loc[hieu_count.person_id.isin(ppl_count)].hieu_id.tolist()
hieu_subset = hieu_count.loc[hieu_count.hieu_id.isin(hieu_match)]
processed_count = hieu_subset.person_id.count()
self.person_count += processed_count
self.weight_total += hieu_subset.p_weight.sum()
person_list = hieu_subset.person_id.dropna().reset_index(drop=True)
print("The firm has: " + str(processed_count) + " number of people with weights: " + str(hieu_subset.p_weight.sum()))
print("Running weight total: " + str(self.weight_total))
self.curr_firm_fake_ind = int(self.Firm[self.Firm.firm_id == self.curr_firm].fake_firm.values)
self.curr_f_choices = np.unique(choice_lst)
self._determine_hieus()
# determine mcaid eligiblity for all persons under adjusted ACA expansion
pop_df = self.Hieu.get_pop_hieu()
pop_df.drop(['fpl','fam_income','family_size'], axis=1, inplace=True)
pop_df = pd.merge(pop_df, self.Family[['family_id','fpl','fam_income','family_size']], on='family_id', how='left')
pop_df.drop_duplicates(inplace=True)
pop_df['new_mcaid_elig'] = self.Mcaid.determine_elig(pop_df)['new_mcaid_elig']
self.Hieu.set_pop_hieu(pop_df)
if self.curr_firm_fake_ind == 0: # a real firm is true
# set a cap on esi contribution levels for firms with size < 100
contr = ['contr_p_self', 'contr_p_sp', 'contr_p_ch', 'contr_p_fam']
sizes = pd.merge(self.Esi[['firm_id', contr[0], contr[1], contr[2], contr[3]]], self.Firm[['firm_id', 'firm_size']], on='firm_id', how='left')
sizes_sset = sizes.loc[sizes.firm_size < 100]
sizes_sset[contr] = sizes_sset[contr].applymap(lambda x: min(x, self.esi_contr_cap))
self.Esi.set_index(['firm_id','choice'], inplace=True)
sizes_sset.set_index('firm_id', inplace=True)
self.Esi.update(sizes_sset)
self.Esi.reset_index(inplace=True)
# initial offers
prem_df = pd.DataFrame({'firm_id' : pd.Series(self.curr_firm), 'choice' : pd.Series(0)})
prem_df = pd.merge(prem_df, self.Esi[['firm_id', 'choice', 'plan_id', 'av_value']], how = 'inner')
self._prev_offer = self._determine_offer(0)
f_size = self.Firm.firm_size[self.Firm.firm_id == self.curr_firm]
f_size = int(f_size)
self.admin_load = self.PremiumParams.admin_load(f_size)
self.EmployerPenalty.set_firm(f, self.Worker)
### firm tendency to continue
if (self._prev_offer == 1):
self.threshold_2_change_cov = float(self.Firm.tendency_to_continue[self.Firm.firm_id == self.curr_firm]) *float(self.firm_threshold_continue_offering)
else:
self.threshold_2_change_cov = float(self.Firm.tendency_to_continue[self.Firm.firm_id == self.curr_firm]) *float(self.firm_threshold_continue_not_offering)
else:
self._prev_offer = 0
### taking exchange premiums as arguments, Dec 30, 2016, xc
### passing a copy of Worker and Family down stream Mar 5, 2017
[docs] def premium_loop(self, xc_prem, worker_table, fam_table, prev_offer, individual_mandate):
"""
loop that calculates premium for ESI plans
"""
if self.track==True:
print("current firm is %d" % self.curr_firm)
if self.curr_firm_fake_ind ==0:
if self.track==True:
print("running premium loop on actual firm %d" % self.curr_firm)
# setup for premium iterations
fchoices = []
for c in self.curr_f_choices:
prem_df = pd.DataFrame({
'firm_id' : pd.Series(self.curr_firm),
'choice' : pd.Series(c)
})
prem_df = pd.merge(
prem_df,
self.Esi[['firm_id', 'choice', 'plan_id', 'av_value', \
'premium_self_actual', 'premium_sp_actual', \
'premium_ch_actual', 'premium_fam_actual']],
how = 'inner'
)
# setup parameters for FirmChoice object
(curr_ch_c, curr_esi_c, c_offer_c, rtio_c) = self._setup_choice(c, prem_df)
# add FirmChoice object to list
fchoices.append(FirmChoice(fam_table, worker_table, curr_ch_c, curr_esi_c, c_offer_c, rtio_c, prem_df))
# iterations of premium loop, c_keys never has the default choice!!!! [xc]
# dictionary of tables from default choice
default_choice_tables = {}
for fch in fchoices:
t0 = time.clock()
if self.track==True:
print("running on: choice %s " % fch.get_curr_choice())
default_choice = (fch.get_curr_choice() == 0)
# fill default_choice_tables with default choice
self._iterate(fch, xc_prem, default_choice, individual_mandate, fchoices[0])
if default_choice and self.calibrateRun == True:
#self._build_long_calib_table(self.long_calib_table)
print("use the original")
if self.long_debug_run and not default_choice:
non_default_long_calib_table = self.Hieu.get_long_calib_table()
non_default_long_calib_table['firm_choice'] = fch.get_curr_choice()
self._build_long_calib_table(non_default_long_calib_table, True)
t1 = time.clock()
if self.debug is True:
print("duration: %f" % (t1-t0))
# default_choice is true/false, if true, use the original fam_table
# if false, use the new_fam_table, where wage has been updated ONCE from wage change
# the original family table should not be modified at all, and the new_fam_table is
# only modified once
# determine firm costs
cost_df = self._accumulate_costs(fchoices)
if self.curr_firm in self.bypass_firms:
chosen_choice = 0
else:
chosen_choice = self._choose_cost(cost_df)
self._reassign_attr(fchoices, chosen_choice) # self.ind_choices is created in this call
self._apply_choice(chosen_choice)
self._set_perm_choices(chosen_choice, 1, self.ind_choices)
self._build_firm_choices_df(chosen_choice, cost_df, self.curr_year)
self._update_curr_esi(chosen_choice)
return (fchoices[chosen_choice].get_worker_table(), fchoices[chosen_choice].get_family_table())
else:
if self.track==True:
print("Running on fake firm %d" % self.curr_firm)
self._determine_hieus()
self.Curr_esi = self._determine_curr_esi(True)
# fake firms do not modify family or worker table; not necessary to return the tables after individual behavior
(self.ind_choices, self.plan_details) = self.Hieu.individual_behavior(self.Curr_esi, self.Esi_0, \
xc_prem, individual_mandate, fam_table, worker_table, self.Firm, self.inc_adjuster)
if self.calibrateRun == 1:
self._build_long_calib_table(self.Hieu.get_long_calib_table())
if self.calibrateRun == 2:
self.output.force_output(1, 'f' + str(self.curr_firm) + "_calib.csv", self.Hieu.get_long_calib_table())
self._set_perm_choices(0, 0, self.ind_choices)
fake_cost_df = pd.DataFrame(np.zeros((1, 6)))
fake_cost_df.columns=['esi_contr','excise_taxes','ft_wage','pt_wage','tax','total_cost']
self._build_firm_choices_df(0, fake_cost_df, self.curr_year)
return (worker_table, fam_table)
#running current choice
def _iterate(self, fch, xc_prem, default_choice, individual_mandate, fch0=None):
"""
requires self.curr_choice and self.c_offer to be made or else just exits
"""
# Manifest Constants
fam_table = fch.get_family_table()
worker_table = fch.get_worker_table()
fch.set_c_offer(self._determine_offer(fch.get_curr_choice()))
if not default_choice:
# preparing for updating wage and income
default_choice_fam_table = fch0.default_choice_fam_table.copy(True)
default_choice_worker_table = fch0.default_choice_worker_table.copy(True)
if default_choice and self._prev_offer:
if self.debug == True:
print("default choice and self._prev_offer")
# created for later, storing the updated wage and income
default_choice_worker_table = worker_table.copy(True).set_index('person_id')
default_choice_fam_table = fam_table.copy(True)
if fch.get_c_offer():
self.ExciseTax.premium_change(self.curr_firm, fch.get_worker_table(), fch.get_prem_df())
for i in range(0, int(self.prem_loop_iter)):
if self.track==True:
print("iteration: " + str(i))
(fch.ind_choices, fch.plan_details) = self.Hieu.individual_behavior(fch.get_curr_esi(), \
self.Esi_0, xc_prem, individual_mandate, fch.get_family_table(), \
fch.get_worker_table(), self.Firm, self.inc_adjuster)
if self.calibrateRun == 1:
self._build_long_calib_table(self.Hieu.get_long_calib_table())
if self.calibrateRun == 2:
self.output.force_output(1, 'f' + str(self.curr_firm) + "_calib.csv", self.Hieu.get_long_calib_table())
if fch.get_c_offer() and i != (int(self.prem_loop_iter)-1):
self._premium_update(fch)
# update Calsim-level ESI
self._update_wage(fch,fam_table,worker_table) # updating wage, due to continue to offer, newly enrollees, from 12 functions
# calculating for wage for other choices, compensating wage
people_df = self.policy_on_curr_firm(fch.ind_choices, self.curr_firm)
people_df = people_df.loc[people_df['fam_income'] > 0] ### subsetting to positive income [xxxx]
### subsetting to those who have chosen ESI, check!
pids = ['p0_id','p1_id','p4_id','p5_id','p6_id','p7_id']
if not people_df.empty:
self.Curr_esi = fch.get_curr_esi()
retain_df = pd.DataFrame({'hieu_id': fch.ind_choices['hieu_id'],'retain_flag':1})
exempt_wrks = fch.get_worker_table()[['person_id', 'cobra_flag', 'eret_flag']]
# start with retain_flag on; while iterating when encountering cobra or eret, make retain_flag = 0
# remove all the rows that should not be retained at the end
for pid in pids:
inddf = pd.DataFrame({'person_id': fch.ind_choices[pid]})
inddf = pd.merge(inddf, exempt_wrks, on='person_id', how='left')
inddf.fillna(0, inplace=True)
inddf[['cobra_flag', 'eret_flag']] = inddf[['cobra_flag','eret_flag']].astype(int)
inddf['not_exempt'] = ((inddf['cobra_flag'] | inddf['eret_flag']) == 0).astype(int)
retain_df['retain_flag'] = retain_df['retain_flag'] * inddf['not_exempt']
elig_choices = pd.merge(fch.ind_choices,retain_df, on='hieu_id', how='left')
elig_choices = elig_choices.loc[elig_choices['retain_flag']==1]
newWage= self._compensate_wages(elig_choices,worker_table,fam_table)
default_choice_fam_table = pd.merge(default_choice_fam_table, newWage[['family_id', 'updatedIncome']], on='family_id', how='left') # updatedIncome is returned
default_choice_fam_table['updatedIncome'].fillna(default_choice_fam_table['fam_income'], inplace=True)
default_choice_fam_table['Change_Income'] = default_choice_fam_table['updatedIncome'] - default_choice_fam_table['fam_income']
default_choice_fam_table.drop('fam_income', axis=1, inplace=True)
default_choice_fam_table.rename(columns={'updatedIncome': 'fam_income'}, inplace=True)
else:
default_choice_fam_table['Change_Income'] = 0
# wage passthrough
default_choice_fam_table['Change_Income'] = default_choice_fam_table['Change_Income'].apply(lambda x: x*self.passthru_dropcov_inc if x > 0 else x*self.passthru_dropcov_dec)
default_choice_fam_table['fpl']=self.Fpl.calculate_fpl(default_choice_fam_table)
# updating worker table,
default_choice_worker_table = pd.merge(default_choice_worker_table , default_choice_fam_table, how='left', on=['family_id'])
default_choice_worker_table ['wages'] = default_choice_worker_table ['wages'] + default_choice_worker_table ['Change_Income']
# return dictionary of tables created from default choice (to be used by other choices later)
# add choice column to worker table
individual_choices = self._wide_to_long(fch.ind_choices, ['p_id'])[['choice', 'p_id']]
individual_choices.rename(columns = {'p_id':'person_id', 'choice':'choice_default'}, inplace=True)
individual_choices = individual_choices.dropna(thresh=2)
default_choice_worker_table.update(individual_choices.set_index('person_id'), overwrite=True)
# update FirmChoice object with default choice tables
fch.default_choice_fam_table = default_choice_fam_table
fch.default_choice_worker_table = default_choice_worker_table
elif default_choice and not self._prev_offer:
if self.debug == True:
print("default choice and not self._prev_offer")
default_choice_worker_table = worker_table.copy(True)
default_choice_fam_table = fam_table.copy(True)
if self.track==True:
print("iteration: 0 ")
(fch.ind_choices, fch.plan_details) = self.Hieu.individual_behavior(fch.get_curr_esi(), \
self.Esi_0, xc_prem, individual_mandate, fch.get_family_table(), \
fch.get_worker_table(), self.Firm, self.inc_adjuster)
if self.calibrateRun == 1:
self._build_long_calib_table(self.Hieu.get_long_calib_table())
if self.calibrateRun == 2:
self.output.force_output(1, 'f' + str(self.curr_firm) + "_calib.csv", self.Hieu.get_long_calib_table())
# add choice column to worker table
individual_choices = self._wide_to_long(fch.ind_choices, ['p_id'])[['choice', 'p_id']]
individual_choices.rename(columns = {'p_id':'person_id', 'choice':'choice_default'}, inplace=True)
individual_choices = individual_choices.dropna(thresh=2)
default_choice_worker_table.update(individual_choices.set_index('person_id'), overwrite=True)
# update FirmChoice object with default choice object
fch.default_choice_fam_table = default_choice_fam_table
fch.default_choice_worker_table = default_choice_worker_table
else: ### other choices following the default choice, self.family should be updated by self.new_fam_table when stops offering
if self.debug:
print("other choices following the default choice")
if fch.get_c_offer():
self.ExciseTax.premium_change(self.curr_firm, fch.get_worker_table(), fch.get_prem_df())
# updating Family and Worker to only those who are no longer eligible for ESI
if self._prev_offer: # default is offering, update wage for full time workers
if int(fch.get_curr_esi()[fch.get_curr_esi().firm_id == self.curr_firm].pt) == 0:
#update worker table
update_pt_worker = default_choice_worker_table[(default_choice_worker_table.full_time == 0) \
& (default_choice_worker_table.firm_id == self.curr_firm)]
if not update_pt_worker.empty:
updated_worker_table = fch.get_worker_table()
updated_worker_table.update(update_pt_worker, overwrite=True)
fch.set_worker_table(updated_worker_table)
#update Family table
temp = pd.merge(fch.get_family_table(), update_pt_worker[update_pt_worker.Change_Income > 0] \
[['Change_Income','family_id']], how='inner', on='family_id')
temp['fam_income'] = temp['fam_income'] + temp['Change_Income']
updated_family_table = fch.get_family_table()
updated_family_table.set_index('family_id', inplace=True)
updated_family_table.update(temp.set_index('family_id'), overwrite=True)
updated_family_table['fpl'] = self.Fpl.calculate_fpl(updated_family_table)
updated_family_table.reset_index(inplace=True)
fch.set_family_table(updated_family_table)
if (not fch.get_c_offer() and self._prev_offer) or (fch.get_c_offer() and not self._prev_offer):
updated_family_table = fch.get_family_table()
updated_worker_table = fch.get_worker_table()
updated_family_table.update(default_choice_fam_table, overwrite=True)
updated_family_table['fpl'] = self.Fpl.calculate_fpl(updated_family_table)
updated_worker_table.update(default_choice_worker_table , overwrite=True)
fch.set_family_table(updated_family_table)
fch.set_worker_table(updated_worker_table)
# using the new compensated wage to start with
for i in range(0, int(self.prem_loop_iter)):
if self.track==True:
print("iteration: " + str(i))
(fch.ind_choices, fch.plan_details) = self.Hieu.individual_behavior(fch.get_curr_esi(), \
self.Esi_0, xc_prem, individual_mandate, fch.get_family_table(),
fch.get_worker_table(), self.Firm, self.inc_adjuster)
if not fch.get_c_offer():
break
elif i != (int(self.prem_loop_iter)-1):
self._premium_update(fch)
if fch.get_c_offer():
self._update_wage(fch,fam_table,worker_table)
if self.calibrateRun == 1:
self._build_long_calib_table(self.Hieu.get_long_calib_table())
if self.calibrateRun == 2:
self.output.force_output(1, 'f' + str(self.curr_firm) + "_calib.csv", self.Hieu.get_long_calib_table())
### premium update based on individual choices
def _premium_update(self, fch):
"""
updates premiums
"""
mrg_df = pd.merge(fch.plan_details, fch.get_prem_df(), on=('firm_id', 'plan_id'), how = 'left')
mrg_df = pd.merge(mrg_df, fch.get_rtio(), how = 'left') # with the same AV, the ratio between self, sp, ch and family plan
# for the time being, assume single plan for each choice,
# will add the code for multiple plans later [xc, 3/29/2017]
# denominator is n_s + c_c*n_c + c_d*n_d+c_f*n_f (Greg's document) for each plan
x = mrg_df[['s_holder', 'sp_holder', 'ch_holder', 'fam_holder']].values
y = mrg_df[['self','sp','ch', 'fam']].values
mrg_df['denom'] = np.multiply(x, y).sum(axis=1)
mrg_df['numor'] = (mrg_df.av_value.values) * (mrg_df.total_exp.values) / (1-self.admin_load) # the toal_exp is passed from individual behavior model
denom_overall = mrg_df['denom'].sum()
num_overall = mrg_df['numor'].sum()
if denom_overall > 0:
new_proposed_self_premium = num_overall/denom_overall
old_self_premium = mrg_df[['premium_self_actual']].min().values
abs_dff = abs(old_self_premium - new_proposed_self_premium)/old_self_premium
if abs_dff <= self.system_parameter.min_prem_thresh:
new_self_premium = new_proposed_self_premium
elif new_proposed_self_premium > old_self_premium:
new_self_premium = old_self_premium*(1 + self.system_parameter.max_prem_thresh)
else:
new_self_premium = old_self_premium*(1 - self.system_parameter.min_prem_thresh)
mrg_df['premium_self_actual'] = new_self_premium
mrg_df['premium_sp_actual'] = new_self_premium*fch.get_rtio()['sp'].values
mrg_df['premium_ch_actual'] = new_self_premium*fch.get_rtio()['ch'].values
mrg_df['premium_fam_actual'] = new_self_premium*fch.get_rtio()['fam'].values
prem_cols = ['firm_id', 'choice', 'plan_id', 'premium_self_actual', 'premium_sp_actual', 'premium_ch_actual', 'premium_fam_actual']
updated_curr_esi = fch.get_curr_esi()
updated_curr_esi.set_index(['firm_id', 'choice', 'plan_id'], inplace = True)
updated_curr_esi.update(mrg_df[prem_cols].set_index(['firm_id', 'choice', 'plan_id']), overwrite = True)
updated_curr_esi.reset_index(inplace = True)
fch.set_curr_esi(updated_curr_esi)
# creating indiviudal choice table when running on a specific firm
# concatenating and removing duplicates???
# more importantly, NEED to bring self.Fam_Table and self.Worker_Table update to the choice made by the firm
def _set_perm_choices(self, choice, real_firm_ind, indata):
# bug fix for missing column when concatenating
if "pholder_0" not in list(indata.columns.values):
for column in ['pholder_' + str(i) for i in range(8) if i != 2 and i != 3]:
indata[column] = None
if 'perm_ind_choices' not in self.__dict__.keys():
self.perm_ind_choices = indata
else:
temp_df = self.perm_ind_choices[~np.in1d(self.perm_ind_choices.hieu_id, indata.hieu_id)]
temp_df = temp_df.reindex()
self.perm_ind_choices = pd.concat([temp_df, indata])
self.perm_ind_choices = self.perm_ind_choices[indata.columns.values.tolist()]
def _inflate_table_year(self, year):
self._build_teleport()
for t in self.inflate_tables.keys():
for c in self.inflate_tables[t]:
if t=='Hieu':
hieu_df = self.Hieu.get_hieu_table()
hieu_df[c]=hieu_df[c]* self._inflate_parameter(year, c)
self.Hieu.set_hieu_table(hieu_df)
else:
self.__dict__[t][c] = self.__dict__[t][c] * self._inflate_parameter(year, c)
def _inflate_parameter(self, year, factor):
year_idx =(self.inflate.year >= self.base_year)& (self.inflate.year <= self.curr_year)
inflator = np.prod(self.inflate[self.inflate_map[factor]][year_idx])
return(inflator)
def _inflate_latent_variables(self, year, calib_year):
if (self.curr_year <= calib_year):
inflator = 1
else:
year_idx = (self.inflate.year > calib_year) & (self.inflate.year<= self.curr_year)
inflator = np.prod(self.inflate[self.inflate_map['latent_vars']][year_idx])
return(inflator)
# reading individual choice table by firm
def _read_choice(self):
"""
used to add current choice to the simulation class
"""
curr_dir = self.choice_path
all_file_names = os.listdir(curr_dir)
firm_string = '^f_' + str(self.curr_firm) + '.(csv|zip)$'
test = re.compile(firm_string)
file_names = filter(test.search, all_file_names)
inf_file_name = 'inf_(%s)cal%d_f_%s.csv' % (self.update_lv, self.use_cal_flag, self.curr_firm)
# read in choice or else raise IOError
if file_names:
if inf_file_name not in all_file_names or self.force_inf_choice:
test = re.compile('.zip$')
zipped_file = filter(test.search, file_names)
if zipped_file:
choices = unzip_to_df(curr_dir + zipped_file[0])
else:
choices = pd.read_csv(curr_dir + file_names[0],memory_map=True)
if (self.exp_type[0]!=0) | (self.exp_type[1] !=0 ):
choices = self._adjust_exp_oop(choices, self.current_path + self.exp_fname)
# replacing latent variables with another set
if self.update_lv is not None:
choices = self._replace_cal_parameters(choices, self.Latent_vars)
if (self.use_cal_flag == 1):
choices = self._update_cal_parameters(choices) # updating new_cal variables to be used in Hieu.py
#choices.to_csv(curr_dir + inf_file_name, index=False)
else:
choices = pd.read_csv(curr_dir + inf_file_name,memory_map=True)
hieu_choices = self._inflate_choice(self.curr_year, choices)
hieu_choices = hieu_choices.fillna(-1)
hieu_choices = hieu_choices.astype(int)
hieu_choices = hieu_choices.replace('-1', np.nan)
self.exp_firm = self._expenditure_out(hieu_choices)
#drop ex-ante medicaid people's other insurance choices if we want to force
#ex-ante medicaid people to stay on medicaid
#update the medicaid calibration parameter based on gvmt_insurance from the hieu table
if self.medicaid_restriction == 1:
hieu_df = self.Hieu.get_hieu_table()
mod_hieu_gvmt_ins = pd.pivot_table(hieu_df, index='hieu_id', columns= 'person_type', values= 'gvmt_insurance')
mod_hieu_gvmt_ins.columns = ['gvmt_ins_0','gvmt_ins_1','gvmt_ins_2','gvmt_ins_3','gvmt_ins_4',\
'gvmt_ins_5','gvmt_ins_6','gvmt_ins_7']
mod_hieu_gvmt_ins = mod_hieu_gvmt_ins.reset_index()
hieu_choices = pd.merge(hieu_choices, mod_hieu_gvmt_ins, on = 'hieu_id', how = 'left')
hieu_choices = self._drop_choice_mcaid('gvmt_ins_0', 'adult_1', hieu_choices)
hieu_choices = self._drop_choice_mcaid('gvmt_ins_1', 'adult_2', hieu_choices)
hieu_choices = self._drop_choice_mcaid('gvmt_ins_2', 'child_1', hieu_choices)
hieu_choices = self._drop_choice_mcaid('gvmt_ins_3', 'child_2', hieu_choices)
hieu_choices = self._drop_choice_mcaid('gvmt_ins_4', 'adult_child_1', hieu_choices)
hieu_choices = self._drop_choice_mcaid('gvmt_ins_5', 'adult_child_2', hieu_choices)
hieu_choices = self._drop_choice_mcaid('gvmt_ins_6', 'adult_child_3', hieu_choices)
hieu_choices = self._drop_choice_mcaid('gvmt_ins_7', 'adult_child_4', hieu_choices)
hieu_choices.drop(['gvmt_ins_0','gvmt_ins_1','gvmt_ins_2','gvmt_ins_3','gvmt_ins_4',\
'gvmt_ins_5','gvmt_ins_6','gvmt_ins_7'], axis=1, inplace = True)
self.Hieu.set_choices(hieu_choices)
else:
self.logger.critical('raising IOError')
raise IOError('cannot find chouce file for firm: ' + str(self.curr_firm))
def _expenditure_out(self, choices):
# return a table of expenditure and oop variables, one line per person
result_df = pd.DataFrame()
for i, person in enumerate(self.persons):
temp_df = choices[[person, 'p'+str(i)+'_id', 'exp'+str(i), 'oop'+str(i), 'exp'+str(i)+'_adj', 'oop'+str(i)+'_adj']]
temp_df.rename(columns={person: 'choice_type', 'p'+str(i)+'_id': 'person_id', 'exp'+str(i): 'exp_average', 'oop'+str(i): 'oop_average', \
'exp'+str(i)+'_adj': 'exp_actual', 'oop'+str(i)+'_adj': 'oop_actual'}, inplace=True)
temp_df = temp_df.drop_duplicates(subset=['choice_type','person_id'])
temp_df.sort_values(['person_id', 'choice_type'], inplace=True)
result_df = result_df.append(temp_df, ignore_index = True) # ignoring index is optional
# reshape the data from long to wide
exp_types = ['exp_average', 'exp_actual', 'oop_average', 'oop_actual']
wide_df = pd.DataFrame()
for typ in exp_types:
temp_df = result_df.pivot_table(index='person_id', columns='choice_type', values=typ)
temp_df.columns = [typ + '_' + str(i) for i in temp_df.columns.values]
wide_df = wide_df.join(temp_df, how='right')
return wide_df.reset_index()
[docs] def propagate_fpl_changes(self, hieu_df, fam_df, worker_df, esi_df, affordability_type):
pop_df0 = self.Hieu.get_pop_hieu()
pop_df = pd.merge(hieu_df, self.Family[['family_id', 'fpl', 'fam_income','family_size', 'tax_filing_status']], \
on='family_id', how='left')
pop_df.drop_duplicates(inplace=True)
if 'mcaid_elig_exante' in pop_df0:
pop_df['mcaid_elig_exante'] = pop_df0['mcaid_elig_exante']
pop_df['predicted_exante_elig'] = pop_df0['predicted_exante_elig']
else:
pop_df['mcaid_elig_exante'] = self.Mcaid.produce_elig_exante(pop_df)
pop_df['predicted_exante_elig'] = (pop_df['mcaid_predict'] >= self.mcaid_exante_threshold) & \
(pop_df['doc_status']<=2) & (pop_df['coverage']!=4)
pop_df = self.Mcaid.determine_elig(pop_df) # re-calculate mcaid eligibility
pop_df = self.Exchange.subsidy_elig(pop_df, worker_df, esi_df, affordability_type) # re-calculate subsidy eligibility
self.Hieu.set_pop_hieu(pop_df)
return pop_df
def _replace_cal_parameters(self, choices, ul_file):
# update the cal columns
for i, person in enumerate(self.persons):
p_info = choices[['hieu_id', person, 'p%d_id' % i]]
p_info = pd.merge(p_info, ul_file, left_on='p%d_id' % i, right_on='person_id', how='left')
cal_accumulator = pd.Series(np.zeros(p_info.shape[0]))
for index, j in enumerate(range(1,11)):
p_mask = p_info[person] == j
cal_accumulator += (p_info[p_info.columns[j+2]].mask(~p_mask)).fillna(0)
choices['cal%d' % i] = cal_accumulator
# second time for new_cal, the second group of ten columns
for index, j in enumerate(range(1,11)):
p_mask = p_info[person] == j
cal_accumulator += (p_info[p_info.columns[j+12]].mask(~p_mask)).fillna(0)
choices['new_cal%d' % i] = cal_accumulator
return choices
def _adjust_exp_oop(self, choices, c_path):
# setup
exp_df = pd.read_csv(c_path,memory_map=True)
choice_df = choices
choice_map = {1: 'unins', 2: 'esi', 3: 'esi', 4: 'esi', 5: 'esi', 6: 'private', 7: 'private', 8: 'private', 9: 'mcaid', 10: 'mcare'}
choice_lst = ['unins', 'esi', 'private', 'mcaid', 'mcare']
choice_cat = {choice: [] for choice in choice_lst}
'''
person_id exp_esi_actual exp_private_actual exp_mcaid_actual exp_mcare_actual exp_unins_actual
oop_esi_actual oop_private_actual oop_mcaid_actual oop_mcare_actual oop_unins_actual
choice exp_esi_average oop_esi_average exp_private_average oop_private_average exp_mcare_average
oop_mcare_average exp_mcaid_average oop_mcaid_average exp_unins_average oop_unins_average
'''
rst_cat = []
if self.exp_type[0] == 1:
for choice in choice_lst:
choice_cat[choice].extend([choice + '_exp_average', choice + '_oop_average'])
rst_cat.extend(['expA', 'oopA'])
if self.exp_type[1] == 1:
for choice in choice_lst:
choice_cat[choice].extend([choice + '_exp_actual', choice + '_oop_actual'])
rst_cat.extend(['expB', 'oopB'])
xc_lst = [(6, [self.exp_xc_multiplier[0], self.oop_xc_multiplier[0]]),
(7, [self.exp_xc_multiplier[1], self.oop_xc_multiplier[1]]),
(8, [self.exp_xc_multiplier[2], self.oop_xc_multiplier[2]])]
for i, person in enumerate(self.persons):
# align people to their corresponding adjusted expenditures and oop-s
temp_df = pd.DataFrame({person: choice_df[person], 'person_id': choice_df['p'+str(i)+'_id']})
temp_df = pd.merge(temp_df, exp_df, on='person_id', how='left')
temp_df['cat'] = temp_df[person].map(choice_map)
rst_df = pd.DataFrame()
# iterate through 5 possible choices
for choice in choice_cat.keys():
# extract updated exp and oop for chosen choice
sset_df = temp_df.loc[temp_df['cat'] == choice]
sset_df = sset_df[choice_cat[choice]].join(sset_df[[person, 'person_id']])
for j, col in enumerate(choice_cat[choice]):
sset_df.rename(columns={col: rst_cat[j]}, inplace=True)
sset_df.drop_duplicates(['person_id', person], inplace=True)
rst_df = rst_df.append(sset_df)
for tup in xc_lst:
if self.exp_type[0] == 1:
rst_df.loc[rst_df[person] == tup[0], 'expA'] *= tup[1][0]
rst_df.loc[rst_df[person] == tup[0], 'oopA'] *= tup[1][1]
if self.exp_type[1] == 1:
rst_df.loc[rst_df[person] == tup[0], 'expB'] *= tup[1][0]
rst_df.loc[rst_df[person] == tup[0], 'oopB'] *= tup[1][0]
# merge back into Hieu dfs
choice_df = pd.merge(rst_df, choice_df, left_on=['person_id', person], right_on=['p'+str(i)+'_id', person], how='right')
choice_df.drop('person_id', inplace=True, axis=1)
if self.exp_type[0] ==1 and self.exp_type[1] ==1:
choice_df.drop(['exp'+str(i), 'oop'+str(i)], inplace=True, axis=1)
choice_df.rename(columns={'expA': 'exp'+str(i), 'oopA': 'oop'+str(i)}, inplace=True)
choice_df.drop(['exp'+str(i)+'_adj', 'oop'+str(i)+'_adj'], inplace=True, axis=1)
choice_df.rename(columns={'expB': 'exp'+str(i)+'_adj', 'oopB': 'oop'+str(i)+'_adj'}, inplace=True)
if self.exp_type[0] ==1 and self.exp_type[1] ==0:
choice_df.drop(['exp'+str(i), 'oop'+str(i)], inplace=True, axis=1)
choice_df.rename(columns={'expA': 'exp'+str(i), 'oopA': 'oop'+str(i)}, inplace=True)
if self.exp_type[0] ==0 and self.exp_type[1] ==1:
choice_df.drop(['exp'+str(i)+'_adj', 'oop'+str(i)+'_adj', 'person_id'], inplace=True, axis=1)
choice_df.rename(columns={'expB': 'exp'+str(i)+'_adj', 'oopB': 'oop'+str(i)+'_adj'}, inplace=True)
choice_df.fillna(0, inplace=True)
return choice_df
def _drop_choice_mcaid(self, gvmt_ins, choice_type, choices):
"""
finds where gvmt_ins_i = 1 and choice_type !=9 and drop these rows
"""
gvmt_ins_idx = np.where((choices[choice_type]!=9) & (choices[gvmt_ins]==1))
choices = choices.drop(choices.index[gvmt_ins_idx[0]])
return(choices)
def _inflate_choice(self, year, choices):
temp_c = choices
for c in self.inflate_choices:
temp_c[c] = temp_c[c] * self._inflate_parameter(year, c)
#inflating the second time to make it var*inflator^2
#for c in ['var0', 'var1', 'var2', 'var3', 'var4', 'var5', 'var6', 'var7']:
# temp_c[c] = temp_c[c] * self._inflate_parameter(year, c)
## hard-coded calibration year for now
for c in self.inflate_latent_variables:
temp_c[c] = temp_c[c] * self._inflate_latent_variables(year, self.calib_year)
return(temp_c)
def _determine_hieus(self):
"""
returns a list of the hieus
"""
ppl = self.Worker.person_id[self.Worker.firm_id == self.curr_firm]
hieu_df = self.Hieu.get_hieu_table()
hieus = hieu_df.hieu_id[np.in1d(hieu_df.person_id.values, ppl)].values
self.curr_hieus = hieus
def _update_cal_parameters(self, choices):
# update the new cal columns
for i, person in enumerate(self.persons):
p_info = choices[['hieu_id', person, 'p%d_id' % i]]
p_info = pd.merge(p_info, self.C_params, left_on='p%d_id' % i, right_on='person_id', how='left')
cal_accumulator = pd.Series(np.zeros(p_info.shape[0]))
for index, j in enumerate(range(1,11)):
p_mask = p_info[person] == j
cal_accumulator += ((p_info[p_info.columns[j+3]].mask(~p_mask)).fillna(0) * self.cal_multipliers[index])
choices['new_cal%d' % i] = cal_accumulator
# update the mcaid predict
force_mcaid = self.C_params[['person_id','mcaid_predict']]
force_mcaid = force_mcaid.loc[force_mcaid['mcaid_predict'] == 1]
force_mcaid.set_index('person_id', inplace=True)
hieu_df = self.Hieu.get_hieu_table().set_index('person_id')
hieu_df.update(force_mcaid)
hieu_df.reset_index(inplace=True)
self.Hieu.set_hieu_table(hieu_df)
''' ## no need to modify original choice tables
try:
choices.to_csv(self.choice_path + file_name, index=False)
except:
print("permission denied: make sure you have permission to modify choice files")
exit()
'''
return choices
def _determine_curr_esi(self, fake):
"""
returns data frame of current esi's
"""
self.logger.debug('determining curr esi\'s')
fake_firm_ids = self.Firm[self.Firm.fake_firm.astype(int) == 1].firm_id.values
fake_firms = self.Esi.firm_id.isin(fake_firm_ids) ### fake firms are not in ESI table, by design
curr_esi = self.Firm[['firm_id', 'curr_choice']]
curr_esi.columns = [['firm_id', 'choice']]
curr_esi = curr_esi[curr_esi['firm_id'] != self.curr_firm]
other_esi = pd.merge(curr_esi, self.Esi, how = 'left', on=('firm_id', 'choice'))
if not fake:
this_esi = self.Esi[(self.Esi.firm_id == self.curr_firm) & (self.Esi.choice == self.curr_choice)]
curr_esi = other_esi.append(this_esi).reset_index(drop=True)
return curr_esi
else:
return other_esi
def _determine_offer(self, choice):
"""
returns if there are offers associated with current choice
if the firm is not a real firm, there won't be an offer associated with the firm
"""
curr_esi = self.Firm[['firm_id', 'curr_choice']] ### always the historical choice
curr_esi.columns = [['firm_id', 'choice']]
curr_esi = curr_esi[curr_esi != self.curr_firm]
this_esi = self.Esi[(self.Esi.firm_id == self.curr_firm) & \
(self.Esi.choice == choice)]
x = this_esi['plan_id'].isnull().all()
return not x
def _setup_choice(self, choice, prem_df):
"""
sets up choice for premium_loop
Curr_esi contains ALL the firms current offering plans
relevant to real firms
If current firm is a fake firm, then it is Curr_esi shouldn't matter [xc]
"""
self.logger.debug('setting up choice ' + str(choice))
self.curr_choice = choice
self.Curr_esi = self._determine_curr_esi(False)
self.c_offer = self._determine_offer(choice)
if self.c_offer:
self.rtio = self.PremiumParams.prem_ratio(prem_df)
else:
self.rtio = None
return (self.curr_choice, self.Curr_esi, self.c_offer, self.rtio)
def _excise_tax(self, fch):
if fch.get_c_offer == 1 and self.curr_year >= self.firm_parameters.excise_tax_year:
curr_plans = self._curr_f_plns()
f_contr_self = fch.ind_choices.s_pln * curr_plans.premium_self_actual * (1 - curr_plans.contr_p_self)
excise_taxes_self = (f_contr_self >= self.firm_parameters.excise_threshold_self) *\
self.firm_parameters.excise_tax_rate *\
(f_contr_self - self.firm_parameters.excise_threshold_self)
f_contr_sp = fch.ind_choices.sp_pln * curr_plans.premium_sp_actual * (1 - curr_plans.contr_p_sp)
excise_taxes_sp = (f_contr_sp>= self.firm_parameters.excise_threshold_sp) *\
self.firm_parameters.excise_tax_rate *\
(f_contr_sp - self.firm_parameters.excise_threshold_sp)
f_contr_ch = fch.ind_choices.ch_pln * curr_plans.premium_ch_actual * (1 - curr_plans.contr_p_ch)
excise_taxes_ch = (f_contr_ch >= self.firm_parameters.excise_threshold_ch) *\
self.firm_parameters.excise_tax_rate *\
(f_contr_ch - self.firm_parameters.excise_threshold_ch)
f_contr_fam = fch.ind_choices.fam_pln * curr_plans.premium_fam_actual * (1 - curr_plans.contr_p_fam)
excise_taxes_fam = (f_contr_fam >= self.firm_parameters.excise_threshold_fam) *\
self.firm_parameters.excise_tax_rate *\
(f_contr_fam - self.firm_parameters.excise_threshold_fam)
f_contr_fam = excise_taxes_self + excise_taxes_sp + excise_taxes_ch + excise_taxes_fam
return f_contr_fam
else:
return np.repeat(0, fch.ind_choices.shape[1])
### when offering, call this function...
def _update_wage(self, fch, fam_table,worker_table):
### fch has information about wage change, using this as a testing group
wage_cols = ['hieu_id', 'wage_delta_0', 'wage_delta_1', 'wage_delta_4', \
'wage_delta_5','wage_delta_6','wage_delta_7']
wage_df = fch.ind_choices[wage_cols]
wagelist = ['wage_delta_0', 'wage_delta_1', 'wage_delta_4', \
'wage_delta_5','wage_delta_6','wage_delta_7']
wage_df['total_change'] = wage_df[wagelist].sum(axis=1)
# wage passthrough
wage_df['total_change'] = wage_df['total_change'].apply(lambda x: x*self.passthru_addcov_inc if x > 0 else x*self.passthru_addcov_dec)
hieu_df = self.Hieu.get_hieu_table()
wage_df = pd.merge(hieu_df[['family_id', 'hieu_id']].drop_duplicates('hieu_id'), wage_df, on='hieu_id', how='left')
wage_df['total_change'].fillna(0, inplace=True)
#temp = pd.merge(self.Family, wage_df.drop(wagelist, axis=1), on='family_id', how='left')
temp = pd.merge(fam_table, wage_df.drop(wagelist, axis=1), on='family_id', how='left')
# get rid of duplicates
to_drop = [col for col in temp if col.endswith('_y')]
temp.drop(to_drop, axis=1, inplace=True)
# rename the columns we want to keep
for col in temp:
if col.endswith('_x'):
temp.rename(columns={col:col.rstrip('_x')}, inplace=True)
temp['total_change'].fillna(0, inplace=True)
temp['fam_income'] = temp['fam_income'] - temp['total_change'] # individuals pay the total change to their premiums
updated_family_table = fch.get_family_table()
updated_family_table.update(temp)
updated_family_table.fpl=self.Fpl.calculate_fpl(updated_family_table)
fch.set_family_table(updated_family_table)
# update worker table
wage_df_long = pd.wide_to_long(wage_df, ['wage_delta_'], i='hieu_id', j ='person_type')
wage_df_long.reset_index(inplace=True)
wage_df_long = pd.merge(wage_df_long, hieu_df[['person_id', 'hieu_id', 'person_type']], on=['hieu_id', 'person_type'], \
how='left')
temp2 = pd.merge(self.Worker, wage_df_long[['person_id', 'total_change']], on='person_id', how='left')
# get rid of duplicates
to_drop = [col for col in temp2 if col.endswith('_y')]
temp2.drop(to_drop, axis=1, inplace=True)
# rename the columns we want to keep
for col in temp2:
if col.endswith('_x'):
temp2.rename(columns={col:col.rstrip('_x')}, inplace=True)
temp2['total_change'].fillna(0, inplace=True)
threshold = pd.DataFrame()
threshold['deducted_wages'] = temp2['wages'] - temp2['total_change']
threshold['min_wages'] = self.MinWage.minwage_threshold(fch.get_worker_table())['threshold']
temp2['wages'] = threshold[['deducted_wages', 'min_wages']].max(axis=1)
updated_worker_table = fch.get_worker_table()
updated_worker_table.update(temp2)
fch.set_worker_table(updated_worker_table)
### dynamic exchange premium
def _cal_exchange_premiums(self, indata, which_exp_to_use, calc_method_flag=0):
"""
input:
indata: dataframe with columns "exp", "age", "type"
calc_method_flag: calculation method flag (0 for multiplier method, 1 for AV-specific method)
which_version: version=1 --> use claims; version=0 --> use AV value
output:
premiums: dictionary as follows -> {6: silver premium, 7: bronze premium, 8: cat premium)
* will return NaN in place of value if no person of that given plan type exists in DataFrame
"""
##########################################################################################################
# MANIFEST CONSTANTS
##########################################################################################################
# Premiums will be returned at the end
if self.dynamic_version == 1:
premiums = {6: 0, 7: 0, 8: 0}
# Administrative loading parameters
admin_load = {6: self.xc_parameters.at[6, 'loading'], 7: self.xc_parameters.at[7, 'loading'], 8: self.xc_parameters.at[8, 'loading']}
# Admin load used for method 1 is currently average of above
admin_load_total = np.mean(admin_load.values())
# Alpha parameters
#alpha = {6: self.xc_parameters.at[6, 'alpha'], 7: self.xc_parameters.at[7, 'alpha'], 8: self.xc_parameters.at[8, 'alpha']}
curr_rp = self.Exchange.get_reg_prem()
state_prem_ratio = self.Exchange.get_state_prem_ratio()
##########################################################################################################
# CALCULATION
##########################################################################################################
# Create Deep Copy of Dataframe, Calculate m(age) for everyone, insert column into DataFrame copy
indata_m_age = indata.copy(True)
#fix problematic data, cannot have exp smaller than oop
indata_m_age["oop"]=np.amin(indata_m_age.loc[:,["exp","oop"]],axis=1)
indata_m_age["oop_adj"]=np.amin(indata_m_age.loc[:,["exp_adj","oop_adj"]],axis=1)
indata_m_age.index = indata['age']
indata_m_age['m_age'] = self.age_rating_parameters['rating']
# get sum(m(agei)) weighted by p_weight and region
indata_m_age.reset_index(drop=True, inplace=True)
indata_m_age["true_subsidy_elig"]=0
indata_m_age.loc[((indata_m_age.xc_sub==1)&(indata_m_age.subsidy_amount_total>0)&\
(indata_m_age.choice.isin([1,2,3,4,5,9,10]))),"true_subsidy_elig"]=1
indata_m_age.loc[((indata_m_age.xc_sub==1)&(indata_m_age.sub_amount>0)&\
(indata_m_age.choice.isin([6,7,8]))),"true_subsidy_elig"]=1
s_m_age_unloaded = indata_m_age[((indata_m_age['choice']==6)&(indata_m_age.true_subsidy_elig==0))].merge(curr_rp[['region', 'mult_slv_uns']], how='left', on='region')
s_m_age_loaded = indata_m_age[((indata_m_age['choice']==6)&(indata_m_age.true_subsidy_elig==1))].merge(curr_rp[['region', 'mult_slv']], how='left', on='region')
b_m_age_unloaded = indata_m_age[((indata_m_age['choice']==7)&(indata_m_age.true_subsidy_elig==0))].merge(curr_rp[['region', 'mult_brz_uns']], how='left', on='region')
b_m_age_loaded = indata_m_age[((indata_m_age['choice']==7)&(indata_m_age.true_subsidy_elig==1))].merge(curr_rp[['region', 'mult_brz']], how='left', on='region')
c_m_age_unloaded = indata_m_age[((indata_m_age['choice']==8)&(indata_m_age.true_subsidy_elig==0))].merge(curr_rp[['region', 'mult_cat_uns']], how='left', on='region')
c_m_age_loaded = indata_m_age[((indata_m_age['choice']==8)&(indata_m_age.true_subsidy_elig==1))].merge(curr_rp[['region', 'mult_cat']], how='left', on='region')
s_m_age_sum = (s_m_age_unloaded['m_age'] * s_m_age_unloaded['p_weight'] * s_m_age_unloaded['mult_slv_uns']).sum()+\
(s_m_age_loaded['m_age'] * s_m_age_loaded['p_weight'] * s_m_age_loaded['mult_slv']).sum()
b_m_age_sum = (b_m_age_unloaded['m_age'] * b_m_age_unloaded['p_weight'] * b_m_age_unloaded['mult_brz_uns']).sum()+\
(b_m_age_loaded['m_age'] * b_m_age_loaded['p_weight'] * b_m_age_loaded['mult_brz']).sum()
c_m_age_sum = (c_m_age_unloaded['m_age'] * c_m_age_unloaded['p_weight'] * c_m_age_unloaded['mult_cat_uns']).sum()+\
(c_m_age_loaded['m_age'] * c_m_age_loaded['p_weight'] * c_m_age_loaded['mult_cat']).sum()
m_age_sum = s_m_age_sum + b_m_age_sum + c_m_age_sum
#depends on ab value, choose exp(oop)/exp(oop)-adj for premium calculation. if ab=1,use exp/oop. If ab=2/3,use exp_adj/oop_adj
if which_exp_to_use == 1:
adj='' #whether we want to use adjusted statistics or not
elif which_exp_to_use == 2 or which_exp_to_use == 3:
adj='_adj'
#No longer use av value
#Instead, directly calculate the claims = Total Expenditure - Max OOP
exp_silver=(indata_m_age[indata_m_age['choice'] == 6]['exp'+adj]*indata_m_age[indata_m_age['choice'] == 6]['p_weight']).sum()
exp_bronze=(indata_m_age[indata_m_age['choice'] == 7]['exp'+adj]*indata_m_age[indata_m_age['choice'] == 7]['p_weight']).sum()
exp_gold=(indata_m_age[indata_m_age['choice'] == 8]['exp'+adj]*indata_m_age[indata_m_age['choice'] == 8]['p_weight']).sum()
oop_silver=(indata_m_age[indata_m_age['choice'] == 6]['oop'+adj]*indata_m_age[indata_m_age['choice'] == 6]['p_weight']).sum()
oop_bronze=(indata_m_age[indata_m_age['choice'] == 7]['oop'+adj]*indata_m_age[indata_m_age['choice'] == 7]['p_weight']).sum()
oop_gold=(indata_m_age[indata_m_age['choice'] == 8]['oop'+adj]*indata_m_age[indata_m_age['choice'] == 8]['p_weight']).sum()
#calculate out the total claims
#for each of the insurance plan, claim_(plan) = exp_(plan) - oop_(plan)(where oop_(plan) has Max OOP considered)
claims_total=exp_silver+exp_bronze+exp_gold-oop_silver-oop_bronze-oop_gold
#calculate the reference premium: statewide bronze unloaded 21
premiums[7] = claims_total / ((1 - admin_load_total) * m_age_sum)
premiums[6] = premiums[7] * state_prem_ratio[0]
premiums[8] = premiums[7] * state_prem_ratio[1]
else:
premiums = {6: 0, 7: 0, 8: 0}
# multipliers
Av = {6: self.xc_parameters.at[6, 'av'], 7: self.xc_parameters.at[7, 'av'], 8: self.xc_parameters.at[8, 'av']}
# Administrative loading parameters
admin_load = {6: self.xc_parameters.at[6, 'loading'], 7: self.xc_parameters.at[7, 'loading'], 8: self.xc_parameters.at[8, 'loading']}
# Admin load used for method 1 is currently average of above
admin_load_total = np.mean(admin_load.values())
# Alpha parameters
alpha = {6: self.xc_parameters.at[6, 'alpha'], 7: self.xc_parameters.at[7, 'alpha'], 8: self.xc_parameters.at[8, 'alpha']}
curr_rp = self.Exchange.get_reg_prem()
##########################################################################################################
# CALCULATION
##########################################################################################################
# Create Deep Copy of Dataframe, Calculate m(age) for everyone, insert column into DataFrame copy
indata_m_age = indata.copy(True)
indata_m_age.index = indata['age']
indata_m_age['m_age'] = self.age_rating_parameters['rating']
# multiplier method (flag = 0)
if calc_method_flag == 0:
# get sum(m(agei)) weighted by p_weight and region
indata_m_age.reset_index(drop=True, inplace=True)
s_m_age = indata_m_age[indata_m_age['choice'] == 6].merge(curr_rp[['region', 'mult_slv']], how='left', on='region')
b_m_age = indata_m_age[indata_m_age['choice'] == 7].merge(curr_rp[['region', 'mult_brz']], how='left', on='region')
c_m_age = indata_m_age[indata_m_age['choice'] == 8].merge(curr_rp[['region', 'mult_cat']], how='left', on='region')
s_m_age_sum = (s_m_age['m_age'] * s_m_age['p_weight'] * s_m_age['mult_slv']).sum()
b_m_age_sum = (b_m_age['m_age'] * b_m_age['p_weight'] * b_m_age['mult_brz']).sum()
c_m_age_sum = (c_m_age['m_age'] * c_m_age['p_weight'] * c_m_age['mult_cat']).sum()
m_age_sum = s_m_age_sum + b_m_age_sum + c_m_age_sum
# calculate A /(1 - d) sum(Ei), divide it by value obtained above, store in dictionary
# depends on ab value, choose exp/exp-adj for premium calculation. if ab=1,use exp. If ab=2/3,use exp_adj
if which_exp_to_use == 1:
exp_silver=(indata_m_age[indata_m_age['choice'] == 6]['exp']*indata_m_age[indata_m_age['choice'] == 6]['p_weight']/self.exp_xc_multiplier[0]).sum()
exp_bronze=(indata_m_age[indata_m_age['choice'] == 7]['exp']*indata_m_age[indata_m_age['choice'] == 7]['p_weight']/self.exp_xc_multiplier[1]).sum()
exp_gold=(indata_m_age[indata_m_age['choice'] == 8]['exp']*indata_m_age[indata_m_age['choice'] == 8]['p_weight'] / self.exp_xc_multiplier[2]).sum()
exp_total=exp_silver+exp_bronze+exp_gold
premiums[7] = Av[7] / (1 - admin_load_total) * exp_total / m_age_sum
elif which_exp_to_use == 2 or which_exp_to_use == 3:
exp_silver = (indata_m_age[indata_m_age['choice'] == 6]['exp_adj'] * indata_m_age[indata_m_age['choice'] == 6]['p_weight'] / self.exp_xc_multiplier[0]).sum()
exp_bronze = (indata_m_age[indata_m_age['choice'] == 7]['exp_adj'] * indata_m_age[indata_m_age['choice'] == 7]['p_weight'] / self.exp_xc_multiplier[1]).sum()
exp_gold = (indata_m_age[indata_m_age['choice'] == 8]['exp_adj'] * indata_m_age[indata_m_age['choice'] == 8]['p_weight'] / self.exp_xc_multiplier[2]).sum()
exp_total = exp_silver + exp_bronze + exp_gold
premiums[7] = Av[7] / (1 - admin_load_total) * exp_total / m_age_sum
# use alpha multipliers to get premiums for other plans
premiums[6] = premiums[7] * alpha[6]
premiums[8] = premiums[7] * alpha[8]
# AV-specific method (flag = 1)
elif calc_method_flag == 1:
# get sum(m(agei))
#s_m_age_sum = indata_m_age[indata_m_age['type'] == 6]['m_age'].sum()
#b_m_age_sum = indata_m_age[indata_m_age['type'] == 7]['m_age'].sum()
#c_m_age_sum = indata_m_age[indata_m_age['type'] == 8]['m_age'].sum()
indata_m_age.reset_index(drop=True, inplace=True)
s_m_age = indata_m_age[indata_m_age['choice'] == 6].merge(curr_rp[['region', 'mult_slv']], how='left',on='region')
b_m_age = indata_m_age[indata_m_age['choice'] == 7].merge(curr_rp[['region', 'mult_brz']], how='left',on='region')
c_m_age = indata_m_age[indata_m_age['choice'] == 8].merge(curr_rp[['region', 'mult_cat']], how='left',on='region')
s_m_age_sum = (s_m_age['m_age'] * s_m_age['p_weight'] * s_m_age['mult_slv']).sum()
b_m_age_sum = (b_m_age['m_age'] * b_m_age['p_weight'] * b_m_age['mult_brz']).sum()
c_m_age_sum = (c_m_age['m_age'] * c_m_age['p_weight'] * c_m_age['mult_cat']).sum()
# calculate A /(1 - d) sum(Ei), divide it by values obtained above, store in dictionary
if which_exp_to_use == 1:
premiums[6] = Av[6] / (1 - admin_load[6]) * ((indata_m_age[indata_m_age['choice'] == 6]['exp']*indata_m_age[indata_m_age['choice'] == 6]['p_weight']).sum()) / s_m_age_sum
premiums[7] = Av[7] / (1 - admin_load[7]) * ((indata_m_age[indata_m_age['choice'] == 7]['exp']*indata_m_age[indata_m_age['choice'] == 7]['p_weight']).sum()) / b_m_age_sum
#temporarily use this for choice 8. Need to switch it back when we have gold
premiums[8] = premiums[7] * alpha[8]
#premiums[8] = Av[8] / (1 - admin_load[8]) * ((indata_m_age[indata_m_age['choice'] == 8]['exp']*indata_m_age[indata_m_age['choice'] == 8]['p_weight']).sum()) / c_m_age_sum
elif which_exp_to_use == 2 or which_exp_to_use == 3:
premiums[6] = Av[6] / (1 - admin_load[6]) * ((indata_m_age[indata_m_age['choice'] == 6]['exp_adj'] *indata_m_age[indata_m_age['choice'] == 6]['p_weight']).sum()) / s_m_age_sum
premiums[7] = Av[7] / (1 - admin_load[7]) * ((indata_m_age[indata_m_age['choice'] == 7]['exp_adj'] *indata_m_age[indata_m_age['choice'] == 7]['p_weight']).sum()) / b_m_age_sum
# temporarily use this for choice 8. Need to switch it back when we have gold
premiums[8] = premiums[7] * alpha[8]
#premiums[8] = Av[8] / (1 - admin_load[8]) * ((indata_m_age[indata_m_age['choice'] == 8]['exp_adj'] *indata_m_age[indata_m_age['choice'] == 8]['p_weight']).sum()) / c_m_age_sum
else:
raise ValueError(str(calc_method_flag) + " is not a valid flag (0 - multiplier, 1 - AV specific)")
# RETURN PREMIUMS
return premiums
def _wide_to_long(self, wide_data, to_melt=[], hieu_specific_cols=[]):
"""
input:
wide_data: wide dataframe
to_melt: list of column names designating those that need to be combined
* for all column in to_melt, wide_data must contain column0-column7
* OR 'column' : [list of 8 columns to combine, in CORRECT order] must
* be in special_col_cases
hieu_specific_cols: list of columns that have values that are the same across
all individuals in a family; they will appear in long_data with column names
of the form hieu_[column_name]
output:
long_data: long dataframe after the conversion finishes in this function
"""
wide_data['ins_p2'] = 0 # harmonizing the ins_* columns
wide_data['ins_p3'] = 0
# subset to real people/choices
### SPECIAL CASES
people = self.persons
special_col_cases = {
'people' : self.persons,
'person_type' : [i for i in range(8)],
'silver_premium' : ['silver_premium_' + str(i) for i in range(8)],
'firm_id' : ['p' + str(i) + '_firm_id' for i in range(8)],
'p_id' : ['p' + str(i) + '_id' for i in range(8)],
'ins_p' : ['ins_p' + str(i) for i in range(8)],
'plan_id' : ['p' + str(i) + '_plan_id' for i in range(8)],
'xc_prem' : ['xc_prem_' + str(i) for i in range(8)],
'xc_prem_un_sub' : ['xc_prem_un_sub_' + str(i) for i in range(8)],
'penalty' : ['penalty_' + str(i) for i in range(8)],
'penalty_orig' : ['penalty_orig_' + str(i) for i in range(8)],
'exp_adj' : ['exp' + str(i) +'_adj' for i in range(8)],
'oop_adj' : ['oop' + str(i) +'_adj' for i in range(8)],
'p_firm_id' : ['p' + str(i) + '_firm_id' for i in range(8)],
'sub_chosen' : ['sub_chosen_' + str(i) for i in range(8)]
}
### initialize base structure with hieu_id, person_type, and choice
long_data = pd.melt(wide_data, ['hieu_id'], special_col_cases['people'], 'person_type', 'choice')
long_data.replace(people, [i for i in range(8)], inplace=True)
long_data.sort_values(['hieu_id', 'person_type'], inplace=True)
long_data.reset_index(inplace=True)
long_data.drop('index', 1, inplace=True)
### go through to_melt list and convert them to long data
for column in to_melt:
# generate columns to combine and melt
try:
column_list = special_col_cases[column]
except:
if column + '0' in list(wide_data.columns.values):
column_list = [column + str(i) for i in range(8)]
elif column + '1' in list(wide_data.columns.values):
column_list = [column + str(i + 1) for i in range(8)]
else:
raise ValueError(column + " does not exist in data")
# melt columns and fix ordering of rows so they match that of long_data
melted_df = pd.melt(wide_data, ['hieu_id'], column_list)
if column in special_col_cases:
melted_df['variable'] = pd.Categorical(melted_df['variable'], special_col_cases[column])
melted_df.sort_values(['hieu_id', 'variable'], inplace=True)
melted_df.reset_index(inplace=True)
# add column to long_data
if column.endswith('_'):
column = column[:-1]
long_data[column] = melted_df['value']
### go through hieu_specific_cols list and convert them to long data
if len(hieu_specific_cols) > 0:
wide_data.index = wide_data['hieu_id']
long_data.index = long_data['hieu_id']
for column in hieu_specific_cols:
long_data["hieu_" + column] = wide_data[column]
wide_data.reset_index(inplace=True, drop=True)
long_data.reset_index(inplace=True, drop=True)
### merge with hieu_table to get the sampling weights
long_data = long_data[np.isfinite(long_data['p_id'])]
hieu_df = self.Hieu.get_hieu_table()
long_data = pd.merge(long_data, hieu_df[['person_id', 'p_weight']], left_on = 'p_id', right_on='person_id')
### return long_data result
long_data.rename(columns={'hieu_region': 'region'}, inplace=True)
return long_data
def _build_long_calib_table(self, indata, non_default_choice=False):
if not non_default_choice:
if 'long_calib_table' not in self.__dict__.keys():
self.long_calib_table = indata
else:
temp_df = self.long_calib_table[~np.in1d(self.long_calib_table.hieu_id, indata.hieu_id)]
temp_df = temp_df.reindex()
self.long_calib_table = pd.concat([temp_df, indata])
self.long_calib_table = self.long_calib_table[indata.columns.values.tolist()]
else:
if 'long_calib_table_not_default' not in self.__dict__.keys():
self.long_calib_table_not_default = indata
else:
self.long_calib_table_not_default = pd.concat([self.long_calib_table_not_default, indata])
self.long_calib_table_not_default = self.long_calib_table_not_default[indata.columns.values.tolist()]
self.long_calib_table_not_default.drop_duplicates(['hieu_id', 'firm_choice'], keep='last')
def _build_firm_choices_df(self, choice, cost_df, year):
"""
in:
choice: chosen choice, as string
choices: list of choices
year: integer for current year
out:
df with appended essential firm information
"""
# To ensure no conflicts in the future
choicesCpy = cost_df.copy(True)
# Prepare choices df
choicesCpy['firm_id'] = self.curr_firm
choicesCpy['chosen_choice'] = 0
choicesCpy['year'] = year
# Moving header items around
new_cols = sorted(list(choicesCpy))
new_cols.remove("firm_id")
new_cols.insert(0, "firm_id")
choicesCpy = choicesCpy[new_cols]
# Mark chosen choice
choicesCpy.set_value(int(choice) if self.curr_firm_fake_ind == 0 else 0, "chosen_choice", 1)
try:
# remove all rows that have correspond to firms that have just been processed
self.firm_choices_all = self.firm_choices_all.loc[~(self.firm_choices_all['firm_id'].isin(choicesCpy['firm_id'].tolist()))]
self.firm_choices_all = self.firm_choices_all.append(choicesCpy, ignore_index=True)
self.firm_choices_all.reset_index(inplace=True, drop=True)
except:
self.firm_choices_all = choicesCpy
## JL: 11/07/2018
## update current esi of firm after it chooses its choice
def _update_curr_esi(self, chosen_choice):
self.Curr_esi.drop(self.Curr_esi[self.Curr_esi.firm_id == self.curr_firm].index, inplace=True)
self.Curr_esi = self.Curr_esi.append(self.Esi[(self.Esi.firm_id == self.curr_firm) & (self.Esi.choice == chosen_choice)]).reset_index(drop=True)
self.Firm.loc[self.Firm['firm_id'] == self.curr_firm,'curr_choice']=chosen_choice
### XC: 1/14/2017
### creating a dataframe with following structure
"""
hieu_id, person_type, member_on_policy
"""
[docs] def policy_on_curr_firm(self, wide_data, firm_id, cols_2_keep=None):
### identifying relevant rows, someone is a policy holder and in the relevant firm
ins_2_pick = ['ins_p0','ins_p1','ins_p4','ins_p5','ins_p6','ins_p7']
firm_col = ['p0_firm_id', 'p1_firm_id', 'p4_firm_id', \
'p5_firm_id', 'p6_firm_id', 'p7_firm_id']
ex_idx = np.where((wide_data[ins_2_pick] > 0) & (wide_data[firm_col] == firm_id))
### subsetting
rtn_df = wide_data.loc[np.unique(ex_idx[0])]
if cols_2_keep == None:
return rtn_df
else:
return rtn_df[cols_2_keep + ins_2_pick]
### creating the input data for _compensating_wages
### this is running a loop across individual hieu, since hieu structure is different
def _compensate_wages(self, wide_data, worker_table,family_table):
"""
increases wages to compensate workers for lack of insurance
"""
### MANIFEST CONSTANTS
df_esi = self.Curr_esi[['firm_id', 'plan_id']][(self.Curr_esi['firm_id']==self.curr_firm)]
esi_drop = False
year = self.curr_year
people = self.persons
ageNameArr = ['age_' + str(i) for i in range(8)]
docTypeArr = ['doc_status_' + str(i) for i in range(8)]
planIdArr = ['p%d_plan_id' % i for i in range(8)]
firmIdArr = ['p%d_firm_id' % i for i in range(8)]
esiPremArr = ['esi_premium_' + str(i) for i in range(8)]
premCapArr = ['silver_premium_' + str(i) for i in range(8)]
person_ids = ['p0_id', 'p1_id', 'p2_id', 'p3_id', 'p4_id', 'p5_id', 'p6_id', 'p7_id']
cols_2_keep = ['silver_premium_' + str(i) for i in range(8)] + firmIdArr + planIdArr + \
esiPremArr + ['hieu_id'] + person_ids
name_df = pd.DataFrame({'age':ageNameArr, 'doc_status': docTypeArr, 'prem_cap': premCapArr, \
'plan_id': planIdArr, 'firm_id': firmIdArr, 'esi_premium': premCapArr})
# TO CHECK
'''
name_df = pd.DataFrame({'age':ageNameArr, 'doc_status': docTypeArr, 'prem_cap': premCapArr, \
'plan_id': planIdArr, 'firm_id': firmIdArr, 'esi_premium': [person + '_silver_premium' for person in people]})
'''
### Utility function for _compensate_wages
def build_fam_lists_column(name_df, raw_df):
iHieu = 0
nFamMember = 8
fam_mem_arr = np.empty([len(raw_df.index)], dtype=object)
for index, row in raw_df.iterrows():
memberLst = []
for i in range (0, nFamMember) :
if row[name_df['age'][i]] is None : continue # conditions for non-existed members
member = [row[name_df['age'][i]], row[name_df['doc_status'][i]], row[name_df['prem_cap'][i]], \
row[name_df['plan_id'][i]], row[name_df['firm_id'][i]], row[name_df['esi_premium'][i]]]
memberLst.append(member)
fam_mem_arr[iHieu] = memberLst
iHieu += 1
return pd.DataFrame({'fam_members': fam_mem_arr})
### Prepare datastructure
CompWageDF = self.policy_on_curr_firm(wide_data, self.curr_firm, cols_2_keep)
### Extract data for each person
hieu_df = self.Hieu.get_hieu_table()
CompWageDF = pd.merge(CompWageDF, hieu_df.loc[hieu_df['person_type'] == 0][['hieu_id', 'family_id']], how='inner', on=['hieu_id'])
nrows = CompWageDF.shape[0]
focal_esi_contr =np.zeros((1, nrows))
focal_person_id = np.zeros((nrows, 1))
for index, row in CompWageDF.iterrows():
for c_index, f in enumerate(firmIdArr):
if np.isnan(row[f]) ==False and int(row[f]) ==self.curr_firm:
focal_esi_contr[:, index] = row[esiPremArr[c_index]]
focal_person_id[index, :] = row[person_ids[c_index]]
focal_person_id = pd.DataFrame(focal_person_id, columns=['person_id'])
focal_person_id.index = focal_person_id.person_id
temp = worker_table[['person_id','wages']]
temp.index = temp.person_id
#focal_person_id['wages'] = temp['wages']
focal_person_id = pd.merge(focal_person_id, temp, on='person_id', how='inner')
focal_person_id.reset_index(drop=True)
focal_person_wage = np.array(focal_person_id['wages'])
focal_esi_contr[np.isnan(focal_esi_contr)] = 0
focal_person_wage[np.isnan(focal_person_wage)] = 0
for i in range(8):
CompWageDF = pd.merge(CompWageDF, hieu_df.loc[hieu_df['person_type'] == i][['age', 'doc_status', 'hieu_id']], how='left', on=['hieu_id'])
CompWageDF.rename(index=str, columns={'age': ageNameArr[i], 'doc_status': docTypeArr[i]}, inplace=True)
### Add fam members list
CompWageDF = CompWageDF.where((pd.notnull(CompWageDF)), None)
fam_members_lst = build_fam_lists_column(name_df, CompWageDF)
fam_members_lst.reset_index(inplace=True, drop=True)
CompWageDF.reset_index(inplace=True, drop=True)
CompWageDF['fam_members'] = fam_members_lst['fam_members']
### Add columns from Family Table
CompWageDF = pd.merge(CompWageDF, family_table[['fam_income', 'tax_filing_status', 'family_size', 'family_id']], how='inner', on=['family_id'])
CompWageDF.rename(index=str, columns={'fam_income': 'income', 'tax_filing_status': 'txfstat'}, inplace=True)
### Add Fpl column
CompWageDF['fpl_base'] = self.Fpl.calculate_income(100, CompWageDF['family_size'], CompWageDF)
### Trim and prepare CompWageDF
CompWageDF = CompWageDF[['family_id', 'income', 'txfstat', 'fpl_base', 'fam_members']]
if (CompWageDF[CompWageDF['income']>0].empty): # only positive income
CompWageDF['updatedIncome'] = None
else:
CompWageDF['txfstat'] = CompWageDF['txfstat'] + '_base'
### Prepare incomeArr
incomeArr = np.array(CompWageDF['income'].values)
### Calculate important constants
stateTaxArr = self.StateTax.calculate_tax_h_df(year, incomeArr, CompWageDF, esi_drop, df_esi).ravel()
fedTaxArr = self.FedTax.calculate_tax_h_df(year, incomeArr, CompWageDF, esi_drop, df_esi).ravel()
#premArr = self.Prem.calculate_premium_h_df(incomeArr, CompWageDF, esi_drop).ravel()
#B = incomeArr - fedTaxArr - stateTaxArr - premArr
B = incomeArr - fedTaxArr - stateTaxArr - focal_esi_contr
### F function
def F(inc, df, year, B) :
#print(inc[72]
esi_drop = True
stateTaxArr = self.StateTax.calculate_tax_h_df(year, inc, df, esi_drop, df_esi).ravel()
#print('state: '+str(stateTaxArr[72])
fedTaxArr = self.FedTax.calculate_tax_h_df(year, inc, df, esi_drop, df_esi).ravel()
#print('fed: '+str(fedTaxArr[72])
premArr = self.Prem.calculate_premium_h_df(inc, df, esi_drop).ravel()
#print('premium: '+str(premArr[72])
A = inc- fedTaxArr - stateTaxArr - premArr - (inc -incomeArr)*.0765
return A - B
### Run Optimization
inc0 = incomeArr*1.2
test_column = F(inc0, CompWageDF, year, B)
#max_iters = [0]
#residuals = []
# def iteration(x, f):
# max_iters[0] += 1
# #residuals.append(abs(f).sum())
methods = 'broyden1,df-sane,krylov,broyden2,anderson,linearmixing,diagbroyden,excitingmixing'.split(',')
# t0 = time.time()
#sol = scipy.optimize.root(F, inc0, method=methods[self.optimization_method], args=(CompWageDF, year, B), tol=1e-5, options={'maxiter':100}, callback=iteration)
sol = scipy.optimize.root(F, inc0, method=methods[self.optimization_method], args=(CompWageDF, year, B), tol=1e-5, options={'maxiter':100})
# t1 = time.time()
# print('krylov optimize time: ' + str(t1-t0)
#
# t0 = time.time()
# sol2 = scipy.optimize.root(F, inc0, method='broyden1', args=(CompWageDF, year, B), tol=1e-5, options={'maxiter':100})
# t1 = time.time()
# print('broyden optimize time: ' + str(t1-t0)
#
# t0 = time.time()
# sol3 = scipy.optimize.root(F, inc0, method='df-sane', args=(CompWageDF, year, B), tol=1e-5, options={'maxiter':100})
# t1 = time.time()
# print('df-sane optimize time: ' + str(t1-t0)
#
# test_df = CompWageDF
# test_df['krylov_results'] = sol.x
# test_df['broyden1_results'] = sol2.x
# test_df['df-sane_results'] = sol3.x
# test_df.to_csv('firm_%i_comp_wage_results.csv' % self.curr_firm)
#try:
# self.max_iters.append(max_iters[0])
#except:
# self.max_iters = [max_iters[0]]
CompWageDF['updatedIncome'] = sol.x
return CompWageDF
# calculating firm cost...
def _curr_f_plns(self, fch):
#f_esi = self.Curr_esi[self.Curr_esi.firm_id == self.curr_firm]
esi = fch.ind_choices[['p0_firm_id', 'p1_firm_id', 'p2_firm_id', 'p3_firm_id',
'p4_firm_id', 'p5_firm_id', 'p6_firm_id', 'p7_firm_id']]
plan = fch.ind_choices[['p0_plan_id', 'p1_plan_id', 'p2_plan_id', 'p3_plan_id',
'p4_plan_id', 'p5_plan_id', 'p6_plan_id', 'p7_plan_id']]
curr_esi_holders = (esi == self.curr_firm)
cnts = curr_esi_holders.sum(axis=1)
cnts[cnts == 0] = 1
curr_plans = np.multiply(plan, curr_esi_holders).sum(axis=1)
curr_plans = pd.DataFrame(curr_plans / cnts)
curr_plans.columns = ['plan_id']
curr_plans.drop_duplicates(inplace=True)
curr_plans = pd.merge(curr_plans, fch.get_curr_esi(), on = 'plan_id', how = 'left')
return curr_plans
def _esi_contr(self, fch):
if fch.get_c_offer() == 1:
curr_plans = self._curr_f_plns(fch)
f_contr_self = fch.plan_details.s_holder * curr_plans.premium_self_actual * (1 - curr_plans.contr_p_self)
f_contr_sp = fch.plan_details.sp_holder * curr_plans.premium_sp_actual * (1 - curr_plans.contr_p_sp)
f_contr_ch = fch.plan_details.ch_holder * curr_plans.premium_ch_actual * (1 - curr_plans.contr_p_ch)
f_contr_fam = fch.plan_details.fam_holder * curr_plans.premium_fam_actual * (1 - curr_plans.contr_p_fam)
f_contr = f_contr_self + f_contr_sp + f_contr_ch + f_contr_fam
return(f_contr, curr_plans.premium_self_actual, curr_plans.premium_sp_actual, curr_plans.premium_ch_actual, curr_plans.premium_fam_actual, curr_plans.av_value)
else:
return (0, None, None, None, None, None)
def _accumulate_costs(self, fchoices):
"""
parameters:
c_keys: list of children instances of calsim to represent choices
assumes:
_choice_c are already built
output:
data frame of costs per choice
notes:
helper function to add up all cost dfs
"""
cost_df=pd.DataFrame()
cost_df['choice']=None
for fch in fchoices:
row_num = fch.get_curr_choice()
cost_df = cost_df.append(self._firm_cost(fch, self.employer_mandate, self.offering_threshold))
cost_df = cost_df.reset_index(drop = True)
cost_df.set_value(row_num,'choice', row_num)
return cost_df
[docs] def wage_total(self, worker_table, firm_id, full_time):
"""
parameters: firm_id: current firm; full_time: True/False calculating full time wages
effect: none
output: total wages for either full time or part time workers
"""
f_wrks = worker_table[worker_table.firm_id == firm_id]
f_wrks = f_wrks[f_wrks.full_time == int(full_time)]
if f_wrks.shape[0] == 0:
return 0
else:
return (f_wrks.wages * f_wrks.in_firm_weight).sum()
def _firm_cost(self, fch, employer_mandate, offering_threshold):
"""
assumes:
agg_df is built into choice
output: total costs associated with firm choice
"""
#get number of people ft and eligible
long_data = self._wide_to_long(fch.ind_choices, ['xc_sub_', 'p_id'])
## mimic the process above to get in_firm_weight
mod_wrk_df = fch.get_worker_table()[['person_id','firm_id', 'full_time', 'in_firm_weight', 'firm_size', 'choice_default']]
mod_wrk_df = mod_wrk_df[mod_wrk_df.firm_id == self.curr_firm]
wk_sub = pd.merge(long_data, mod_wrk_df, left_on='p_id', right_on = 'person_id', how = 'left')
people_penalty = wk_sub.loc[(wk_sub["full_time"] == 1) & (wk_sub['firm_id']==self.curr_firm) \
& (wk_sub['xc_sub'] ==1) & (wk_sub['choice'].isin([6,7,8])) , ['in_firm_weight']]
num_penalty = people_penalty['in_firm_weight'].sum()
people_exempt = wk_sub.loc[ (wk_sub["full_time"] == 1) & (wk_sub['choice_default'].isin([3,4,5,9,10])), ['in_firm_weight']]
num_exempt = people_exempt['in_firm_weight'].sum()
people_on_plan = wk_sub.loc[(wk_sub['firm_id']==self.curr_firm) & (wk_sub["full_time"] == 1) & (wk_sub["choice"] == 2) , ['in_firm_weight']]
num_on_plan = people_on_plan['in_firm_weight'].sum()
people_full_time = wk_sub.loc[(wk_sub["full_time"] == 1) & (wk_sub['firm_id']==self.curr_firm), ['in_firm_weight']]
num_full_time = people_full_time['in_firm_weight'].sum()
if (num_full_time - num_exempt) > 0:
pct_takeup = num_on_plan/(num_full_time - num_exempt)
else:
pct_takeup = None
temp = self._esi_contr(fch)
if self._prev_offer == 0 and (fch.get_c_offer() != 0) and ((pct_takeup < offering_threshold) | (pct_takeup ==None)):
data = {
'tax' : 0,
'pt_wage' : 0,
'ft_wage' : 0,
'esi_contr' : 0,
'excise_taxes': 0,
'firm_latent' : 0,
}
cost_df = pd.DataFrame(data, index = [0])
cost_df['total_cost'] = -99999
cost_df['av_value'] = temp[5]
else:
if (employer_mandate ==1):
#add frim_latents for default choice
if fch.get_curr_choice()==0:
if self.firm_latents_path is not None:
data = {
'tax':self.FirmTax.total_tax(self.curr_firm, self.Firm, fch.get_worker_table()),
'pt_wage':self.wage_total(fch.get_worker_table(), self.curr_firm, False),
'ft_wage':self.wage_total(fch.get_worker_table(), self.curr_firm, True),
'employer_penalty':self.EmployerPenalty.penalty(fch.get_c_offer(), num_penalty, self.fplty_mult),
'esi_contr':temp[0],
'excise_taxes':self._excise_tax(fch).sum(),
'firm_latents': self.firm_latents[self.firm_latents['firm_id']==self.curr_firm].latent.max(),}
else:
data = {
'tax': self.FirmTax.total_tax(self.curr_firm, self.Firm, fch.get_worker_table()),
'pt_wage': self.wage_total(fch.get_worker_table(), self.curr_firm, False),
'ft_wage': self.wage_total(fch.get_worker_table(), self.curr_firm, True),
'employer_penalty': self.EmployerPenalty.penalty(fch.get_c_offer(), num_penalty,self.fplty_mult),
'esi_contr': temp[0],
'excise_taxes': self._excise_tax(fch).sum()
}
else:
data = {
'tax': self.FirmTax.total_tax(self.curr_firm, self.Firm, fch.get_worker_table()),
'pt_wage': self.wage_total(fch.get_worker_table(), self.curr_firm, False),
'ft_wage': self.wage_total(fch.get_worker_table(), self.curr_firm, True),
'employer_penalty': self.EmployerPenalty.penalty(fch.get_c_offer(), num_penalty, self.fplty_mult),
'esi_contr': temp[0],
'excise_taxes': self._excise_tax(fch).sum(),
}
else:
#add firm_latens for default choice
if fch.get_curr_choice() == 0:
if self.firm_latents_path is not None:
data = {
'tax':self.FirmTax.total_tax(self.curr_firm, self.Firm, fch.get_worker_table()),
'pt_wage':self.wage_total(fch.get_worker_table(), self.curr_firm, False),
'ft_wage':self.wage_total(fch.get_worker_table(), self.curr_firm, True),
'esi_contr':temp[0],
'excise_taxes':self._excise_tax(fch).sum(),
'firm_latent': self.firm_latents[self.firm_latents['firm_id'] == self.curr_firm].latent.max(),
}
else:
data = {
'tax': self.FirmTax.total_tax(self.curr_firm, self.Firm, fch.get_worker_table()),
'pt_wage': self.wage_total(fch.get_worker_table(), self.curr_firm, False),
'ft_wage': self.wage_total(fch.get_worker_table(), self.curr_firm, True),
'esi_contr': temp[0],
'excise_taxes': self._excise_tax(fch).sum()
}
else:
data = {
'tax': self.FirmTax.total_tax(self.curr_firm, self.Firm, fch.get_worker_table()),
'pt_wage': self.wage_total(fch.get_worker_table(), self.curr_firm, False),
'ft_wage': self.wage_total(fch.get_worker_table(), self.curr_firm, True),
'esi_contr': temp[0],
'excise_taxes': self._excise_tax(fch).sum(),
}
cost_df = pd.DataFrame(data, index = [0])
cost_df['total_cost'] = cost_df[cost_df.columns].sum(axis = 1)
cost_df['pct_takeup'] = pct_takeup
cost_df['num_full_time'] = num_full_time
cost_df['premium_self_actual'] = temp[1]
cost_df['premium_sp_actual'] = temp[2]
cost_df['premium_ch_actual'] = temp[3]
cost_df['premium_fam_actual'] = temp[4]
cost_df['av_value'] = temp[5]
cost_df['offer'] = fch.get_c_offer()
cost_df['prev_offer'] = self._prev_offer
return cost_df
def _choose_cost(self, cost_df):
"""
input:
cost_df has been calculated
output:
the choice (as a string) that is the optiminal (minimum) choice
"""
cost_df = cost_df.loc[(cost_df["total_cost"] >= 0)]
cost_df.reset_index(drop=True, inplace=True)
temp =cost_df[cost_df['choice']==0]
default_choice_cost=temp['total_cost'].iloc[0]
min_cost = cost_df[cost_df['choice'] > 0 ]['total_cost'].min()
# Return either default choice, or a random minimum choice if conditions below hold
choice = 0
if (min_cost >=0) and (min_cost < default_choice_cost) and \
(abs((default_choice_cost - min_cost) / default_choice_cost) > self.threshold_2_change_cov):
choice = cost_df[cost_df['total_cost'] == min_cost]['choice'].iloc[0]
return choice
def _apply_choice(self, choice):
"""
input:
choice: the _choice_c of the chosen choice
effect:
change Firm current choice with chosen choice
"""
try:
c = int(choice)
self.Firm[self.Firm['firm_id'] == self.curr_firm]['curr_choice'] = c
except ValueError:
# TODO add in logging methods
return
def _reassign_attr(self, fchoices, choice):
"""
input:
choice: the _choice_c of the chosen choice
effect:
takes Family and Worker attribute of child (_choice_c) and reasigns to parent (self)
"""
choice_types = self.persons
# for chosen choice, update the individual choice table
self.ind_choices = fchoices[choice].ind_choices
#self.Family = fchoices[choice].get_family_table() # updating family table
#self.Worker = fchoices[choice].get_worker_table() # updating worker table
phold_0= self.ind_choices[choice_types[0]]==2
phold_0_dependents_2 = (self.ind_choices[choice_types[1:]]==3).sum(axis=1)
phold_0_dependents_3 = (self.ind_choices[choice_types[1:]]==4).sum(axis=1)
#multiply boolean if person has insurance type 2; if not, then policy holder element is 0
self.ind_choices['pholder_0'] = phold_0*(phold_0_dependents_2+phold_0_dependents_3+phold_0)
phold_1= self.ind_choices[choice_types[1]]==2
phold_1_dependents_2 = self.ind_choices[choice_types[0]]==3
phold_1_dependents_3 = (self.ind_choices[choice_types[2:]]==5).sum(axis=1)
self.ind_choices['pholder_1'] = phold_1*(phold_1_dependents_2+phold_1_dependents_3+phold_1)
self.ind_choices['pholder_4'] = (self.ind_choices[choice_types[4]]==2).astype(int)
self.ind_choices['pholder_5'] = (self.ind_choices[choice_types[5]]==2).astype(int)
self.ind_choices['pholder_6'] = (self.ind_choices[choice_types[6]]==2).astype(int)
self.ind_choices['pholder_7'] = (self.ind_choices[choice_types[7]]==2).astype(int)
[docs] def cleanup(self):
"""
function that resets certain variables after each year
"""
del self.firm_choices_all
del self.perm_ind_choices
def _build_teleport(self):
for t in self.teleport_tables:
self.__dict__[t] = type(self.__dict__[self.teleport_tables[t]])(
self.__dict__[self.teleport_tables[t]].copy()
)