Merge pull request #3 from THEOLEX-IO/report

data_processing.py

@@ -43,7 +43,7 @@ def load_data():
 def process_data(data):
     decisions = pd.DataFrame(data['decisions'])
     decisions['year'] = pd.to_datetime(decisions['decision_date']).dt.year
-    decisions.monetary_sanction = decisions.monetary_sanction.astype(float)
+    decisions.monetary_sanction = pd.to_numeric(decisions.monetary_sanction, errors='coerce').fillna(0)
     # keep validated decisions
     decisions = decisions[decisions.status == 'V']
     decisions.decision_date = pd.to_datetime(decisions['decision_date']).dt.date
@@ -63,7 +63,6 @@ def process_data(data):
     decisions = decisions.merge(organizations, left_on='organizations', right_on='org_id')
     # remove Individual
     decisions = decisions[decisions.org_company_type != "Individual"]
-
     # work on authorities
     authorities = pd.DataFrame(data['authorities'])
     authorities.index = authorities.url.apply(get_id)
@@ -79,12 +78,14 @@ def process_data(data):
 def get_monetary_dataframe(decision_scope):
     monetary_decision = decision_scope[decision_scope.monetary_sanction > 0]
     monetary_decision['has_revenues'] = (monetary_decision.org_revenues != "")
-    monetary_decision['org_revenues'] = monetary_decision.org_revenues.str.replace('', '0').astype(float)
+    monetary_decision['org_revenues'] = pd.to_numeric(monetary_decision.org_revenues, errors='coerce').fillna(0)
     monetary_decision['log10_org_revenues'] = monetary_decision.org_revenues.apply(lambda x: np.log10(x+1))
     monetary_decision['log10_monetary_sanction'] = monetary_decision.monetary_sanction.apply(lambda x: np.log10(x+1))
     monetary_decision['same_country'] = (monetary_decision.org_country == monetary_decision.authorities_country)
     monetary_decision['monetary_sanction_rate'] = monetary_decision.monetary_sanction/monetary_decision.org_revenues
     monetary_decision['log10_monetary_sanction_rate'] = monetary_decision.monetary_sanction_rate.apply(np.log10)
+    time = round((monetary_decision.decision_date - monetary_decision.decision_date.min()) / np.timedelta64(1, "M"))
+    monetary_decision['time'] = time
     return monetary_decision
 
 

model.py

@@ -1,3 +1,4 @@
+import itertools
 import numpy as np
 import pandas as pd
 import xgboost as xgb
@@ -5,48 +6,56 @@ from xgboost import cv
 from sklearn.model_selection import train_test_split
 
 
-def prepare_data(monetary_decision):
+def prepare_predictors(monetary_decision, col_num, col_cat):
     monetary_decision = monetary_decision.reset_index(drop=True)
-    time = round((monetary_decision.decision_date - monetary_decision.decision_date.min()) / np.timedelta64(1, "M"))
-    monetary_decision.loc[:, ('time')] = time
-    col_num = ['log10_org_revenues',
-               'time']
-    col_cat = ['authorities_country',
-               'type',
-               'violation_theme',
-               'justice_type',
-               'org_country',
-               'org_currency',
-               'org_continent',
-               'same_country',
-               'org_company_type']
     predictors = monetary_decision[col_num + col_cat]
-    target = monetary_decision.log10_monetary_sanction
     for col in col_cat:
         predictors[col] = predictors[col].astype("category")
+    return predictors
+
+
+def prepare_data(monetary_decision, col_num, col_cat):
+    predictors = prepare_predictors(monetary_decision, col_num, col_cat)
+    target = monetary_decision.log10_monetary_sanction
     return predictors, target
 
 
 def split(predictors, target):
     predictors_train, predictors_test, target_train, target_test = train_test_split(predictors,
-                                                                          target,
-                                                                          test_size=0.2,
-                                                                          random_state=42)
+                                                                                    target,
+                                                                                    test_size=0.2,
+                                                                                    random_state=50)
     return predictors_train, predictors_test, target_train, target_test
 
 
-def run_training(predictors_train, predictors_test):
-    data_train = xgb.DMatrix(predictors_train, label=predictors_test, enable_categorical=True)
-    params = {'max_depth': 4,
-             'learning_rate': 0.05,
-             'colsample_bytree': 0.3,
-             'subsample': 0.8,
-             'gamma': 0.5,
-             'objective': 'reg:squarederror'}
-    num_round = 1000
-    #xgb_cv = cv(dtrain=data_train, params=params, nfold=3,
-    #            num_boost_round=1000, early_stopping_rounds=10, metrics="rmse", as_pandas=True, seed=123)
-    return xgb.train(params, data_train, num_round)
+def run_cv_training(predictors_train, target_train):
+    data_train = xgb.DMatrix(predictors_train, label=target_train, enable_categorical=True)
+    xgb_csv = []
+    best_params = (100, {}, 10)
+    for eta, max_depth, col_num in itertools.product([0.05, 0.01], [10, 15], [0.3, 0.8]):
+        prefix = f"{str(eta)}_{str(max_depth)}_{str(col_num)}"
+        params = {
+            'learning_rate': eta,
+            'max_depth': max_depth,
+            'colsample_bytree': col_num,
+            # 'gamma': 0.5,
+            'subsample': 0.8,
+            'objective': 'reg:squarederror'}
+        cv_results = cv(dtrain=data_train, params=params, nfold=2,
+                        num_boost_round=1000, early_stopping_rounds=3, metrics="rmse", as_pandas=True, seed=50)
+        best_value = cv_results['test-rmse-mean'].values[-1]
+        best_round = cv_results.index[-1]
+        xgb_csv.append(
+            cv_results.rename(columns={col: f'{prefix}_{col}' for col in cv_results.columns}).tail(10).reset_index())
+        if best_value < best_params[0]:
+            best_params = (best_value, params, best_round)
+
+    return pd.concat(xgb_csv, axis=1), best_params
+
+
+def run_training(predictors_train, target_train, params, num_rounds):
+    data_train = xgb.DMatrix(predictors_train, label=target_train, enable_categorical=True)
+    return xgb.train(params, data_train, num_rounds)
 
 
 def predict(model, predictors):
@@ -55,4 +64,4 @@ def predict(model, predictors):
 
 
 def features_importance(model):
-    return pd.Series(model.get_score(importance_type='gain')).sort_values()
+    return pd.Series(model.get_score(importance_type='gain')).sort_values()

requirements.txt

@@ -12,6 +12,7 @@ certifi==2021.5.30
 cffi==1.14.6
 charset-normalizer==2.0.6
 click==7.1.2
+coverage==6.1.1
 cycler==0.10.0
 debugpy==1.5.0
 decorator==5.1.0
@@ -20,12 +21,14 @@ entrypoints==0.3
 gitdb==4.0.7
 GitPython==3.1.24
 idna==3.2
+iniconfig==1.1.1
 ipykernel==6.4.1
 ipython==7.28.0
 ipython-genutils==0.2.0
 ipywidgets==7.6.5
 jedi==0.18.0
 Jinja2==3.0.2
+joblib==1.1.0
 jsonschema==4.0.1
 jupyter-client==7.0.6
 jupyter-core==4.8.1
@@ -51,31 +54,45 @@ pexpect==4.8.0
 pickleshare==0.7.5
 Pillow==8.3.2
 plotly==5.3.1
+pluggy==1.0.0
+pprintpp==0.4.0
 prometheus-client==0.11.0
 prompt-toolkit==3.0.20
 protobuf==3.18.1
 ptyprocess==0.7.0
+py==1.10.0
 pyarrow==5.0.0
+pycountry==20.7.3
+pycountry-convert==0.7.2
 pycparser==2.20
 pydeck==0.7.0
 Pygments==2.10.0
+Pympler==0.9
 pyparsing==2.4.7
 pyrsistent==0.18.0
+pytest==6.2.5
+pytest-cov==3.0.0
+pytest-mock==3.6.1
 python-dateutil==2.8.2
 pytz==2021.3
 pyzmq==22.3.0
+repoze.lru==0.7
 requests==2.26.0
+scikit-learn==1.0.1
 scipy==1.7.1
 seaborn==0.11.2
 Send2Trash==1.8.0
 six==1.16.0
+sklearn==0.0
 smmap==4.0.0
 statsmodels==0.13.0
-streamlit==0.89.0
+streamlit==1.2.0
 tenacity==8.0.1
 terminado==0.12.1
 testpath==0.5.0
+threadpoolctl==3.0.0
 toml==0.10.2
+tomli==1.2.2
 toolz==0.11.1
 tornado==6.1
 traitlets==5.1.0
@@ -87,5 +104,5 @@ watchdog==2.1.6
 wcwidth==0.2.5
 webencodings==0.5.1
 widgetsnbextension==3.5.1
-xgboost
-sklearn
+xgboost==1.5.0
+sklearn

score_app.py

@@ -2,6 +2,7 @@
 import streamlit as st
 import requests
 import pandas as pd
+import numpy as np
 import datetime
 from data import headers
 
@@ -18,7 +19,7 @@ created_at = st.sidebar.date_input('Date input', value=datetime.date(2021, 1, 1)
 @st.cache
 def load_data(source_type, start_date):
     def get_decision_hist(d_id):
-        url = f"https://www.theolex.io/data/decisions/{d_id}/return_hist/"
+        url = f"https://www.theolex.io/data/decisions/{int(d_id)}/return_hist/"
         res = requests.get(url, headers=headers)
         return res.json()
 
@@ -34,7 +35,8 @@ def load_data(source_type, start_date):
     data_sources = data_sources[data_sources.created_at >= start_date]
 
     # get decisions history
-    data_list = [(_id, get_decision_hist(_id)) for _id in data_sources['decision_id']]
+    # can be optimized by filtering first on validated decision for decision table
+    data_list = [(_id, get_decision_hist(_id)) for _id in data_sources['decision_id'] if not np.isnan(_id)]
     return [(_id, pd.DataFrame(pd.DataFrame(data).fields.to_dict()).T)
             for _id, data in data_list if len(data) > 0]
 

stream_app.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import pandas as pd
 import streamlit as st
+from scipy import stats
 
 import plotly.express as px
 import plotly.figure_factory as ff
@@ -8,7 +9,7 @@ import plotly.figure_factory as ff
 import scipy
 import numpy as np
 from data_processing import load_data, process_data, get_monetary_dataframe, get_themes_per_year
-from model import prepare_data, run_training, split, predict, features_importance
+from model import prepare_predictors, prepare_data, run_training, split, predict, features_importance, run_cv_training
 
 
 def _max_width_():
@@ -44,7 +45,7 @@ else:
     select_auth = authorities.name.sort_values()
 
 authority = st.sidebar.selectbox('Authority', ['All', *select_auth])
-min_year, max_year = st.sidebar.slider('Decisions year', min_value=2001, max_value=2021, value=(2001, 2021))
+min_year, max_year = st.sidebar.slider('Decisions year', min_value=2001, max_value=2021, value=(2008, 2021))
 
 # apply filters
 authority_filter = True
@@ -57,7 +58,7 @@ decision_scope = decisions[authority_filter & year_filter]
 
 st.subheader("Dataset Description")
 
-st.metric('Number of validated decisions liked to organisations (and not individuals)', decision_scope.shape[0])
+st.metric('Number of validated decisions linked to organisations (and not individuals)', decision_scope.shape[0])
 
 st.metric('Decisions with monetary sanctions',
           decision_scope[decision_scope.monetary_sanction > 0].shape[0])
@@ -157,36 +158,63 @@ with st.expander("Data exploration"):
                   width=1000, height=600)
     st.plotly_chart(fig)
 
-
 ##############################################
 ####
 # build ML model
 ####
 ##############################################
 st.title("Training phase")
-
-predictors, target = prepare_data(monetary_decision)
+xgb_model = None
+col_num_all = ['log10_org_revenues',
+               'time']
+col_cat_all = ['authorities_country',
+               'type',
+               'violation_theme',
+               'justice_type',
+               'org_country',
+               'org_currency',
+               'org_continent',
+               'same_country',
+               'org_company_type']
+
+st.sidebar.title("Training params")
+col_num = st.sidebar.multiselect('Numeric variables',
+                                 col_num_all, col_num_all)
+col_cat = st.sidebar.multiselect('Categorical variables',
+                                 col_cat_all, col_cat_all)
 # train the model
+predictors, target = prepare_data(monetary_decision, col_num, col_cat)
 if st.button('Run training'):
     with st.expander("Training results"):
+        # Study distribution
         st.write(f"dataset size: {monetary_decision.shape[0]}")
-        st.markdown("Plot taget distribution: log 10 of monetary sanctions")
-        fig = ff.create_distplot([target], [' log 10 of monetary sanctions'], bin_size=0.1)
+        st.markdown("Plot target distribution: log 10 of monetary sanctions")
+        fig = ff.create_distplot([target], ['log 10 of monetary sanctions'], bin_size=0.1)
         fig.update_layout(width=1000,
                           template="simple_white",
                           height=600,
                           bargap=0.01)
         st.plotly_chart(fig)
 
-        # split data set
+        # Split data set
         predictors_train, predictors_test, target_train, target_test = split(predictors, target)
         st.subheader("Split dataset between training and test:")
         st.metric(label="Training size", value=predictors_train.shape[0])
         st.metric(label="Test size", value=predictors_test.shape[0])
 
-        xgb_model = run_training(predictors_train, target_train)
+        # Run cross validation
+        st.subheader("Cross validation error")
+        with st.spinner('Wait for it...'):
+            xgb_cv, best_params = run_cv_training(predictors_train, target_train)
+
+        st.line_chart(xgb_cv[[col for col in xgb_cv.columns if "mean" in col]])
+        st.subheader("Selected variables")
+        st.json(best_params)
+
+        # Train final model
+        xgb_model = run_training(predictors_train, target_train, best_params[1], best_params[2])
 
-        # evaluate model error
+        # Evaluate model error
         target_train_predicted = predict(xgb_model, predictors_train)
         training_bias = np.mean(target_train_predicted - target_train)
         st.metric(label="Training bias", value=training_bias)
@@ -196,7 +224,7 @@ if st.button('Run training'):
         test_bias = np.mean(test_errors)
         st.metric(label="Test bias", value=test_bias)
 
-        fig = ff.create_distplot([test_errors], ['errors distribution'], bin_size=0.1)
+        fig = ff.create_distplot([test_errors], ['errors distribution'], bin_size=0.2)
         fig.update_layout(width=1000,
                           template="simple_white",
                           height=600,
@@ -251,8 +279,21 @@ if st.button('Run training'):
         R_sq = corr_matrix[0, 1] ** 2
         st.metric(label="Explained variation thanks to model (R^2)", value=f"{round(100 * R_sq, 2)}%")
 
-st.sidebar.title("Organizations view")
-col_x = ['log10_org_revenues', 'authorities_country', 'violation_theme', 'org_country', 'org_company_type']
-sample_revenues = st.sidebar.number_input('Yearly revenues', value=1000000)
-authority = st.sidebar.selectbox('Organization country', predictors.org_country.cat.categories)
-authority = st.sidebar.selectbox('Organization activity', predictors.org_company_type.cat.categories)
+        st.subheader("Residuals & homoscedasticity")
+        # st.metric(label="Explained variation thanks to model (R^2)", value=f"{round(100 * R_sq, 2)}%")
+
+        print(stats.pearsonr(test_errors, target_test))
+
+st.title("Organizations view")
+col1, col2, col3 = st.columns(3)
+to_predict = {}
+with col1:
+    to_predict['log10_org_revenues'] = [np.log10(st.number_input('Yearly revenues', value=100000000))]
+    for col in col_cat:
+        to_predict[col] = [st.selectbox(f'{col}', predictors[col].cat.categories)]
+    print(to_predict)
+
+    df_to_predict = prepare_predictors(pd.DataFrame.from_dict(to_predict), col_num, col_cat)
+    if xgb_model:
+        predicted = predict(xgb_model, df_to_predict)
+        print(predicted)