#!/usr/bin/env python
# utility modules
from utility_modules.math_functions import expand_grid as expand_grid
from utility_modules.get_parameters import get_params
from utility_modules.dotMap import DotMap
from utility_modules.input import Input
# policy modules
from policy_modules.mcaid_elig import McaidEligibility as McaidEligibility
from policy_modules.exchange import Exchange as Exchange
# calsim class tables
from src.FamilyTable import Family as Family
from src.HieuTable import Hieu as Hieu
from src.WorkerTable import Worker as Worker
from src.FirmTable import Firm as Firm
from src.CalSim import CalSim as CalSim
# python libraries
import os
import hashlib
import numpy as np
import pandas as pd
import logging
import argparse
import time
import datetime
import warnings
from itertools import chain
import requests
from requests.exceptions import HTTPError
from tempfile import NamedTemporaryFile
from urllib.error import URLError
import pdb
import configparser
from ast import literal_eval
[docs]def main():
# YOLO -- Kevin's footprint -- You only live once ?
warnings.filterwarnings('ignore')
# gather command line arguments
cmd_parse = argparse.ArgumentParser(description = 'CalSim 2.0', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# general
cmd_parse.add_argument('-config', '--configuration_file', required = True, help = 'A .ini file with necessary configuration information')
cmd_parse.add_argument('-pol_in_opt', '--policy_input_option', type=str, default = None, help = 'Specify a choice of policy input, as defined in config file')
cmd_parse.add_argument('-years', '--years_to_run', help='List of years for the simulation to run.')
cmd_parse.add_argument('-opt_method', '--optimization_method', default = 1, help = 'Algorithm used for optimization. Supports broyden1, \
df-sane, krylov, broyden2, df-sane, anderson, linearmixing, diagbroyden, \
excitingmixing. Use array indexing to choose method.')
# for custom generation of choice file (output from read_choice)
cmd_parse.add_argument('-force_inf_choice', '--force_inflate_choice', action='store_true', help = 'overwrite cached inflated choice csvs')
# for custom generation of partial firm choice in Esi table
cmd_parse.add_argument('-partial_firm_choice', '--partial_firm_choice', action='store_true', help='manually set non-offering firms only have default choice')
# flag for predicted mcaid eligibility
cmd_parse.add_argument('-use_pred_mcaid', '--use_pred_mcaid', action='store_true',help='manually set whether we use predicted mcaid eligibility,default false')
# flag for applying ken's method
cmd_parse.add_argument('-ken','--use_ken_method', action='store_true', help='apply ken method weight adjuster')
# config file duplicates
cmd_parse.add_argument('-d', '--debug', action = 'store_true', help = 'Enable debug mode')
cmd_parse.add_argument('-track', '--track',default=False, action='store_true', help='printing running progress')
cmd_parse.add_argument('-p', '--path', help = 'Path for alternative input directory')
cmd_parse.add_argument('-t', '--test', action = 'store_true', help = 'Enable subsetting of firms for processing')
cmd_parse.add_argument('-suffix', '--suffix', type = str, help = 'Suffix to output folder name')
cmd_parse.add_argument('-user', '--user_name', help = 'User name')
cmd_parse.add_argument('-firm_pass', '--firm_pass', help = 'Firm threshold-pass through parameters: drop: inc, dec, add: inc, dec') # default = '.90,.90,.90,.90'
cmd_parse.add_argument('-u', '--utility_parms', help = 'Parameters for utility tuning, in the order of (esi, sub, unsub, mc, cal, penalty') # default = '1,1,1,1,1,1'
cmd_parse.add_argument('-exp', '--exp_multiplier', help = 'Expenditure multiplier in the order of: (esi, private, mcare, mcaid, unins)') # default = '1,1,1,1,1'
cmd_parse.add_argument('-fp', '--flatp', help = 'Flat penalty amount in the order of: (undoc, doc+exempted, doc+unexempted)') # default = '0,0,0'
cmd_parse.add_argument('-xc_it', '--xc_it', help = 'Dynamic exchange premium, number of iterations. If > 1, then dynamic exchange premium will be assumed.') # default = '1'
cmd_parse.add_argument('-xc_inf', '--xc_inf', help = 'Exchange premium inflator') # default = '1'
cmd_parse.add_argument('-xc_tol', '--xc_premium_tolerance', help = 'tolerance for xc iteration stop criterion')
# cmd-line directory options
cmd_parse.add_argument('-r', '--prefix', help = 'prefix to append in front of timestamp in output folder name')
cmd_parse.add_argument('-c', '--choice_files_path', help = "directory to choice files")
# simulation parameter tuning
cmd_parse.add_argument('-fpm', '--fplty_mult', default = 1, help = 'Firm penalty multiplier')
cmd_parse.add_argument('-pth', '--pt_hrs_esi_elig', help='part time hours threshold for Esi eligibility')
cmd_parse.add_argument('-savings', '--savings', help = 'Saving needed to change coverage type, in the format of \'0.03,0.03\', equivalent to firm_threshold from config file')
cmd_parse.add_argument('-dual', '--dual', default = False, action = 'store_true', help = 'Allow dual eligibility - both exchange subsidy eligible and medicaid eligible')
cmd_parse.add_argument('-pmc', '--pmc', action = 'store_true', help = 'Flag for including predicted medicaid eligibility as part of mcaid eligibility in \
calculating the subsidy eligibility: 1 (True) = treat predicted medicaid eligibility the same\
in calculating exchange subsidy eligibility; 0 (Flase) = only use rigirous subsidy definition. \
Therefore, there would be people with dual eligibility: both exchange subsidy eligible and mcaid \
predicted eligible.')
cmd_parse.add_argument('-ab', '--ab', default=2, help = 'Choose between A (rf prediction) or B (rf prediction on the adjusted expenditure) \
for utility and premium calculation: \
1 = A for utility and A for premium \
2 = A for utility and B for premium 3 = B for utility and B for premium')
cmd_parse.add_argument('-inc', '--inc_adjuster', default = '0,1,0,1,0', help = 'Income adjuster for premium in the order of (on/off flag, ceiling, floor, percent, sensitivity).')
cmd_parse.add_argument('-uexp', '--update_exp', default='0,0', help = 'Toggle for updating exp and oop via external predictive models. \
0,0 = neither the -- actual -- nor the -- average -- will be updated, \
0,1 = only the -- average -- is being updated, \
1,0 = only the -- actual -- is being updated, \
1,1 = both the --actual-- and the --average -- will be updated. \
Assuming the external file specified by expfname option in the input_tables folder with header as: \
esi_exp,esi_oop,esi_exp_adj,esi_oop_adj,\
private_exp,private_oop,private_exp_adj,private_oop_adj,\
mcaid_exp,mcaid_oop,mcaid_exp_adj,mcaid_oop_adj, \
unins_exp,unins_oop,unins_exp_adj,unins_oop_adj, \
mcare_exp,mcare_oop,mcare_exp_adj,mcare_oop_adj,person_id,hieu_id')
cmd_parse.add_argument('-expfname', '--expfname', type=str, default=None, help = 'Specifying the file name used for updating expenditure variables. Assuming that \
it is located in the same location in input_tables table of the crrent data source.')
cmd_parse.add_argument('-risk', '--risk', default = 0.000464, help='Risk aversion parameter, default value from RAND')
cmd_parse.add_argument('-alpha', '--alpha', default = 1, help='Multiplier for expenditure in utility model, RAND set at .3.')
# output
cmd_parse.add_argument('-ld', '--long_debug', default = False, help = 'Very long table in debug mode, including all firm choices. \
This currently does not work when running on the entire set of firms. [2/25/2018]')
cmd_parse.add_argument('-log', '--log_errors', action = 'store_true', help='flag for logging')
cmd_parse.add_argument('-ul', '--update_lv', type = str, default = None, help='File name for replacing calibration latent variable. \
Assuming the external file in the input_tables folder with header ordered as:\
c_unins c_esi_self c_esi_spouse c_esi_parent_1 c_esi_parent_2 \
c_silver c_bronze c_cat c_mcaid c_mcare \
c_unins_adj c_esi_self_adj c_esi_spouse_adj \
c_esi_parent_1_adj c_esi_parent_2_adj c_silver_adj c_bronze_adj c_cat_adj \
c_mcaid_adj c_mcare_adj person_id ')
cmd_parse.add_argument('-fm', '--firm_default', action = 'store_true', help='sets firm choices to default choices to produce firm_default_utility_table.csv')
# Policy part
cmd_parse.add_argument('-afford_def', '--affordability_definition', type=int, help='affordability definition to use 0:single 1:family 2:hybrid')
cmd_parse.add_argument('-firm_thresh', '--firm_threshold',help='firm threshold for firm to switch from offering to non-offering and from non-offering to offering')
cmd_parse.add_argument('-mandate', '--individual_mandate', help='individual_mandate parameter 1 is using individual mandate 0 is not')
cmd_args = cmd_parse.parse_args()
# read in .ini configuration file for simulation, system, and policy information
config = configparser.ConfigParser()
config.read(cmd_args.configuration_file)
config_dict = {}
for section in config.sections():
config_dict[section] = {}
for key, value in config[section].iteritems():
config_dict[section][key] = literal_eval(config[section].get(key))
if cmd_args.policy_input_option is not None:
config_dict['policy_input_option'] = cmd_args.policy_input_option
elif config_dict['General']['policy_input_option'] is not None:
config_dict['policy_input_option'] = config_dict['General']['policy_input_option']
else:
config_dict['policy_input_option'] = 'default'
# setup params dict
params_dict = config_dict['Output']
params_dict = DotDict(params_dict)
# setup policy levers dict
if cmd_args.individual_mandate is not None:
config_dict['Policy']['individual_mandate']=cmd_args.individual_mandate
if cmd_args.affordability_definition is not None:
config_dict['Policy']['affordability_definition']=cmd_args.affordability_definition
if cmd_args.firm_threshold is not None:
config_dict['Policy']['firm_threshold']=map(float, cmd_args.firm_threshold.split(','))
if config_dict['policy_input_option']=='p2':
config_dict['Policy']['bronze_subsidy'] = True
policy_levers_dict = config_dict['Policy']
policy_levers_dict = DotDict(policy_levers_dict)
# setup calibration dict
cal_levers_dict = config_dict['Calibration']
cal_levers_dict = DotDict(cal_levers_dict)
# pass it to calsim
cal_levers_dict['policy_input_option'] = config_dict['policy_input_option']
# setup system levers dict
system_dict = config_dict['System_Sheet']
system_dict = DotDict(system_dict)
# setup path
if cmd_args.user_name is not None:
config_dict['General']['username'] = cmd_args.user_name
if cmd_args.suffix is not None:
config_dict['General']['suffix'] = cmd_args.suffix
if cmd_args.pt_hrs_esi_elig is not None:
config_dict['General']['pt_hrs_esi_elig'] = cmd_args.pt_hrs_esi_elig
config_dict['General']['force_inf_choice'] = cmd_args.force_inflate_choice
#Temporary deactivate force_inf_choice
config_dict['General']['force_inf_choice'] = True
if cmd_args.partial_firm_choice ==True:
config_dict['General']['partial_firm_choice'] = cmd_args.partial_firm_choice
try:
config_dict['General']['partial_firm_choice']
except :
config_dict['General']['partial_firm_choice']=False
if cmd_args.use_ken_method ==True:
config_dict['General']['ken'] = cmd_args.use_ken_method
try:
config_dict['General']['ken']
except :
config_dict['General']['ken']=False
if cmd_args.use_pred_mcaid == True:
config_dict['General']['use_pred_mcaid'] = cmd_args.use_pred_mcaid
try:
config_dict['General']['use_pred_mcaid']
except:
config_dict['General']['use_pred_mcaid'] = False
# general dictionary
general_dict = config_dict['General']
general_dict = DotDict(general_dict)
general_dict.config_path = cmd_args.configuration_file
# updating config file using command line information; over-writing what is in the config file
# variables: debug path test suffix username utility_params firm_pass exp_mult flatp
if cmd_args.years_to_run is not None:
system_dict['years_to_run'] = map(int, cmd_args.years_to_run.split(','))
if cmd_args.debug == True:
params_dict['debug'] = cmd_args.debug
if cmd_args.track == True:
params_dict['track'] = cmd_args.track
if cmd_args.path is not None:
general_dict['datapath'] = cmd_args.path
if cmd_args.test == True:
general_dict['test'] = cmd_args.test
if cmd_args.suffix is not None:
general_dict['suffix'] = cmd_args.suffix
if cmd_args.user_name is not None:
general_dict['username'] = cmd_args.user_name
if cmd_args.xc_it is not None:
system_dict['xc_it'] = cmd_args.xc_it
else:
system_dict['xc_it'] = 1
if cmd_args.xc_premium_tolerance is not None:
system_dict['xc_premium_tolerance'] = cmd_args.xc_premium_tolerance
if cmd_args.utility_parms is not None:
cal_levers_dict['utility_params'] = map(float, cmd_args.utility_parms.split(','))
if cmd_args.firm_pass is not None:
temp = map(float, cmd_args.firm_pass.split(','))
system_dict['passthru_dropcov_inc'] = temp[0]
system_dict['passthru_dropcov_dec'] = temp[1]
system_dict['passthru_addcov_inc'] = temp[2]
system_dict['passthru_addcov_dec'] = temp[3]
if cmd_args.exp_multiplier is not None:
cal_levers_dict['exp_multiplier'] = map(float, cmd_args.exp_multiplier.split(','))
if cmd_args.flatp is not None:
cal_levers_dict['flat_penalties'] = map(float, cmd_args.flatp.split(','))
if cmd_args.firm_default is True:
cal_levers_dict['firm_default'] = True
# updating variables only in command line
config_dict['prefix'] = cmd_args.prefix
general_dict['optimization_method'] = cmd_args.optimization_method
general_dict['long_debug'] = bool(cmd_args.long_debug)
general_dict['pmc_flag'] = cmd_args.pmc
cal_levers_dict['dual'] = int(cmd_args.dual)
cal_levers_dict['ab_choice'] = int(cmd_args.ab)
cal_levers_dict['risk'] = float(cmd_args.risk)
cal_levers_dict['alpha'] = float(cmd_args.alpha)
cal_levers_dict['exp_type'] = map(int, cmd_args.update_exp.split(','))
if cmd_args.expfname is not None:
cal_levers_dict['exp_fname'] = cmd_args.expfname # file name for replacing expenditure variables
cal_levers_dict['inc_adjuster'] = map(float, cmd_args.inc_adjuster.split(','))
if cmd_args.update_lv is not None:
cal_levers_dict['update_lv'] = cmd_args.update_lv # file name for replacing latent variables
if cmd_args.savings is not None:
system_dict['savings'] = map(float, cmd_args.savings.split(','))
system_dict['fplty_mult'] = cmd_args.fplty_mult # flag for how to use predicted medicaid eligibility
# handle logs directorys
logger = logging.getLogger('main')
logger.setLevel(logging.DEBUG)
# create file handler which logs even debug messages
curr_time = time.time()
# initialize other input variables
input_helper = Input(config_dict, general_dict['pol_input_path'])
policy_dict = input_helper.policy_dict
path_dict = input_helper.path_dict
if cmd_args.log_errors and not os.path.exists(os.path.join(config_dict['General']['outpath'], path_dict['Out_path'])):
os.makedirs(os.path.join(config_dict['General']['outpath'], path_dict['Out_path']))
f_name = os.path.join(config_dict['General']['outpath'], path_dict['Out_path'], 'logs.log') if cmd_args.log_errors else '/dev/null'
fh = logging.FileHandler(f_name)
formatter = logging.Formatter('[%(asctime)s %(name)s]: %(message)s')
fh.setFormatter(formatter)
# add the handlers to the logger
logger.addHandler(fh)
if cmd_args.log_errors:
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
ch.setFormatter(formatter)
logger.addHandler(ch)
# validate command line path is valid
if config_dict['General']['datapath']:
logger.debug('path set to: ' + config_dict['General']['datapath'])
if not os.path.isdir(config_dict['General']['datapath']):
raise(IOError('Could not find the user specified path: ' + config_dict['General']['datapath']))
for f in path_dict.keys():
if f == "Out_path" and config_dict['General']['outpath'] != None:
path_dict[f] = os.path.join(config_dict['General']['outpath'], path_dict[f])
if not os.path.exists(path_dict[f]):
try:
os.makedirs(path_dict[f])
except:
raise (IOError('Could not creat output in the user specified path: ' + path_dict[f]))
path_dict[f] = os.path.join(config_dict['General']['datapath'], path_dict[f])
else:
logger.debug('please specify an datapath in config files')
raise Exception('please specify an datapath in config files')
if 'update_lv' in cal_levers_dict.keys():
if not os.path.isfile(cal_levers_dict['update_lv'] ):
raise (IOError('Could not find update_lv file in the user specified path: ' + cal_levers_dict['update_lv']))
if cal_levers_dict['use_cal_flag']==1:
if 'calibration_file' in cal_levers_dict.keys():
if not os.path.isfile(cal_levers_dict['calibration_file'] ):
raise (IOError('Could not find calibration file in the user specified path: ' + cal_levers_dict['calibration_file']))
else:
raise Exception('please specify an calibration file in config files')
# loop through exchange inflator list
#pdb.set_trace()
#list_inf = map(float, cmd_args.xc_inflators.split(','))
#for i in range(len(list_inf)):
if (cmd_args.xc_inf >=0):
policy_levers_dict.xc_premium_inflator = cmd_args.xc_inf
# start simulation
cs = CalSim(path_dict, policy_dict, params_dict, cal_levers_dict, policy_levers_dict, system_dict, general_dict)
cs.start_simulation()
if __name__ == '__main__':
main()