add predict sample

data_processing.py

@@ -43,7 +43,7 @@ def load_data():
 def process_data(data):
     decisions = pd.DataFrame(data['decisions'])
     decisions['year'] = pd.to_datetime(decisions['decision_date']).dt.year
-    decisions.monetary_sanction = decisions.monetary_sanction.astype(float)
+    decisions.monetary_sanction = pd.to_numeric(decisions.monetary_sanction, errors='coerce').fillna(0)
     # keep validated decisions
     decisions = decisions[decisions.status == 'V']
     decisions.decision_date = pd.to_datetime(decisions['decision_date']).dt.date
@@ -63,7 +63,6 @@ def process_data(data):
     decisions = decisions.merge(organizations, left_on='organizations', right_on='org_id')
     # remove Individual
     decisions = decisions[decisions.org_company_type != "Individual"]
-
     # work on authorities
     authorities = pd.DataFrame(data['authorities'])
     authorities.index = authorities.url.apply(get_id)
@@ -79,12 +78,14 @@ def process_data(data):
 def get_monetary_dataframe(decision_scope):
     monetary_decision = decision_scope[decision_scope.monetary_sanction > 0]
     monetary_decision['has_revenues'] = (monetary_decision.org_revenues != "")
-    monetary_decision['org_revenues'] = monetary_decision.org_revenues.str.replace('', '0').astype(float)
+    monetary_decision['org_revenues'] = pd.to_numeric(monetary_decision.org_revenues, errors='coerce').fillna(0)
     monetary_decision['log10_org_revenues'] = monetary_decision.org_revenues.apply(lambda x: np.log10(x+1))
     monetary_decision['log10_monetary_sanction'] = monetary_decision.monetary_sanction.apply(lambda x: np.log10(x+1))
     monetary_decision['same_country'] = (monetary_decision.org_country == monetary_decision.authorities_country)
     monetary_decision['monetary_sanction_rate'] = monetary_decision.monetary_sanction/monetary_decision.org_revenues
     monetary_decision['log10_monetary_sanction_rate'] = monetary_decision.monetary_sanction_rate.apply(np.log10)
+    time = round((monetary_decision.decision_date - monetary_decision.decision_date.min()) / np.timedelta64(1, "M"))
+    monetary_decision['time'] = time
     return monetary_decision
 
 

model.py

@@ -6,25 +6,17 @@ from xgboost import cv
 from sklearn.model_selection import train_test_split
 
 
-def prepare_data(monetary_decision):
+def prepare_predictors(monetary_decision, col_num, col_cat):
     monetary_decision = monetary_decision.reset_index(drop=True)
-    time = round((monetary_decision.decision_date - monetary_decision.decision_date.min()) / np.timedelta64(1, "M"))
-    monetary_decision.loc[:, ('time')] = time
-    col_num = ['log10_org_revenues',
-               'time']
-    col_cat = ['authorities_country',
-               'type',
-               'violation_theme',
-               'justice_type',
-               'org_country',
-               'org_currency',
-               'org_continent',
-               'same_country',
-               'org_company_type']
     predictors = monetary_decision[col_num + col_cat]
-    target = monetary_decision.log10_monetary_sanction
     for col in col_cat:
         predictors[col] = predictors[col].astype("category")
+    return predictors
+
+
+def prepare_data(monetary_decision, col_num, col_cat):
+    predictors = prepare_predictors(monetary_decision, col_num, col_cat)
+    target = monetary_decision.log10_monetary_sanction
     return predictors, target
 
 
@@ -46,7 +38,7 @@ def run_cv_training(predictors_train, target_train):
             'learning_rate': eta,
             'max_depth': max_depth,
             'colsample_bytree': col_num,
-            #'gamma': 0.5,
+            # 'gamma': 0.5,
             'subsample': 0.8,
             'objective': 'reg:squarederror'}
         cv_results = cv(dtrain=data_train, params=params, nfold=2,

stream_app.py

@@ -9,7 +9,7 @@ import plotly.figure_factory as ff
 import scipy
 import numpy as np
 from data_processing import load_data, process_data, get_monetary_dataframe, get_themes_per_year
-from model import prepare_data, run_training, split, predict, features_importance, run_cv_training
+from model import prepare_predictors, prepare_data, run_training, split, predict, features_importance, run_cv_training
 
 
 def _max_width_():
@@ -158,45 +158,63 @@ with st.expander("Data exploration"):
                   width=1000, height=600)
     st.plotly_chart(fig)
 
-
 ##############################################
 ####
 # build ML model
 ####
 ##############################################
 st.title("Training phase")
-
-predictors, target = prepare_data(monetary_decision)
+xgb_model = None
+col_num_all = ['log10_org_revenues',
+               'time']
+col_cat_all = ['authorities_country',
+               'type',
+               'violation_theme',
+               'justice_type',
+               'org_country',
+               'org_currency',
+               'org_continent',
+               'same_country',
+               'org_company_type']
+
+st.sidebar.title("Training params")
+col_num = st.sidebar.multiselect('Numeric variables',
+                                 col_num_all, col_num_all)
+col_cat = st.sidebar.multiselect('Categorical variables',
+                                 col_cat_all, col_cat_all)
 # train the model
+predictors, target = prepare_data(monetary_decision, col_num, col_cat)
 if st.button('Run training'):
     with st.expander("Training results"):
+        # Study distribution
         st.write(f"dataset size: {monetary_decision.shape[0]}")
-        st.markdown("Plot taget distribution: log 10 of monetary sanctions")
-        fig = ff.create_distplot([target], [' log 10 of monetary sanctions'], bin_size=0.1)
+        st.markdown("Plot target distribution: log 10 of monetary sanctions")
+        fig = ff.create_distplot([target], ['log 10 of monetary sanctions'], bin_size=0.1)
         fig.update_layout(width=1000,
                           template="simple_white",
                           height=600,
                           bargap=0.01)
         st.plotly_chart(fig)
 
-        # split data set
+        # Split data set
         predictors_train, predictors_test, target_train, target_test = split(predictors, target)
         st.subheader("Split dataset between training and test:")
         st.metric(label="Training size", value=predictors_train.shape[0])
         st.metric(label="Test size", value=predictors_test.shape[0])
 
-        # run cross validation
+        # Run cross validation
         st.subheader("Cross validation error")
-        xgb_cv, best_params = run_cv_training(predictors_train, target_train)
+        with st.spinner('Wait for it...'):
+            xgb_cv, best_params = run_cv_training(predictors_train, target_train)
 
         st.line_chart(xgb_cv[[col for col in xgb_cv.columns if "mean" in col]])
         st.subheader("Selected variables")
         st.json(best_params)
 
-        # train final model
+        # Train final model
         xgb_model = run_training(predictors_train, target_train, best_params[1], best_params[2])
 
-        # evaluate model error
+        # Evaluate model error
         target_train_predicted = predict(xgb_model, predictors_train)
         training_bias = np.mean(target_train_predicted - target_train)
         st.metric(label="Training bias", value=training_bias)
@@ -261,16 +279,21 @@ if st.button('Run training'):
         R_sq = corr_matrix[0, 1] ** 2
         st.metric(label="Explained variation thanks to model (R^2)", value=f"{round(100 * R_sq, 2)}%")
 
+        st.subheader("Residuals & homoscedasticity")
+        # st.metric(label="Explained variation thanks to model (R^2)", value=f"{round(100 * R_sq, 2)}%")
 
-        st.subheader("Plot predicted vs real")
-        #st.metric(label="Explained variation thanks to model (R^2)", value=f"{round(100 * R_sq, 2)}%")
-
-        print(stats.pearsonr(test_errors, target_test_predicted))
-
+        print(stats.pearsonr(test_errors, target_test))
 
+st.title("Organizations view")
+col1, col2, col3 = st.columns(3)
+to_predict = {}
+with col1:
+    to_predict['log10_org_revenues'] = [np.log10(st.number_input('Yearly revenues', value=100000000))]
+    for col in col_cat:
+        to_predict[col] = [st.selectbox(f'{col}', predictors[col].cat.categories)]
+    print(to_predict)
 
-st.sidebar.title("Organizations view")
-col_x = ['log10_org_revenues', 'authorities_country', 'violation_theme', 'org_country', 'org_company_type']
-sample_revenues = st.sidebar.number_input('Yearly revenues', value=1000000)
-authority = st.sidebar.selectbox('Organization country', predictors.org_country.cat.categories)
-authority = st.sidebar.selectbox('Organization activity', predictors.org_company_type.cat.categories)
+    df_to_predict = prepare_predictors(pd.DataFrame.from_dict(to_predict), col_num, col_cat)
+    if xgb_model:
+        predicted = predict(xgb_model, df_to_predict)
+        print(predicted)