init

data_processing.py

@@ -0,0 +1,73 @@
+import streamlit as st
+import requests
+import pandas as pd
+
+
+# util functions
+def get_id(x):
+    return int(x.split("/")[-2])
+
+
+def get_dict(df, col):
+    return df[col].to_dict()
+
+
+def replace_lis_val(df, col):
+    def _replace(l):
+        return [get_dict(df, col)[i] for i in l]
+
+    return _replace
+
+
+def mode(lst):
+    if lst:
+        return max(set(lst), key=lst.count)
+
+
+@st.cache
+def load_data():
+    url = "https://www.theolex.io/data"
+    validated_filter = "status=V"
+    linked_dataset = "include[]=violations.*&include[]=organizations.*&include[]=authorities.*"
+    url_d = f"{url}/decisions/?per_page=4000&{validated_filter}&{linked_dataset}"
+    response = requests.get(url_d, headers={'authorization': 'Token 8d55a74628aee8122b7a5a1a51f7caad6d613ec1',
+                                            'accept': 'application/json'})
+
+    # work on decisions
+    return response.json()
+
+
+def process_data(data):
+    decisions = pd.DataFrame(data['decisions'])
+    decisions['year'] = pd.to_datetime(decisions['decision_date']).dt.year
+    decisions.monetary_sanction = decisions.monetary_sanction.astype(float)
+    decisions = decisions[decisions.status == 'V']
+    decisions.decision_date = pd.to_datetime(decisions['decision_date']).dt.date
+    decisions['id'] = decisions.url.apply(get_id)
+    decision_col = ['violations', 'authorities', 'organizations', 'country_of_violation', 'type', 'justice_type',
+                    'defendant', 'decision_date', 'monetary_sanction', 'nature_de_sanction', 'violation_theme', 'year']
+    decisions = decisions[decision_col]
+    decisions = decisions.explode('organizations')
+
+    # work on organisations
+    organizations = pd.DataFrame(data['organizations'])
+    organizations['id'] = organizations.url.apply(get_id)
+    organizations = organizations[["id", "name", "company_type", "revenues", "currency", "country", "lei"]]
+    organizations.columns = ['org_' + col for col in organizations.columns]
+    decisions = decisions.merge(organizations, left_on='organizations', right_on='org_id')
+
+    # remove Individual
+    decisions = decisions[decisions.org_company_type != "Individual"]
+    decisions.org_country = decisions.org_country.str.lower().str.strip()
+
+    # work on authorities
+    authorities = pd.DataFrame(data['authorities'])
+    authorities.index = authorities.url.apply(get_id)
+    authorities = authorities[["country", "type", "name"]]
+    countries = {'FR': 'france', 'US': 'United States', 'UK': 'United Kingdom', 'GE': 'Germany'}
+    authorities.country = authorities.country.apply(lambda v: countries.get(v, v)).str.lower().str.strip()
+
+    decisions['authorities_name'] = decisions.authorities.apply(replace_lis_val(authorities, 'name'))
+    decisions['authorities_country'] = decisions.authorities.apply(replace_lis_val(authorities, 'country')).apply(mode)
+
+    return decisions, organizations, authorities

requirements.txt

@@ -1,7 +1,89 @@
+altair==4.1.0
+argon2-cffi==21.1.0
+astor==0.8.1
+attrs==21.2.0
+backcall==0.2.0
+backports.zoneinfo==0.2.1
+base58==2.1.0
+bleach==4.1.0
+blinker==1.4
+cachetools==4.2.4
+certifi==2021.5.30
+cffi==1.14.6
+charset-normalizer==2.0.6
+click==7.1.2
+cycler==0.10.0
+debugpy==1.5.0
+decorator==5.1.0
+defusedxml==0.7.1
+entrypoints==0.3
+gitdb==4.0.7
+GitPython==3.1.24
+idna==3.2
+ipykernel==6.4.1
+ipython==7.28.0
+ipython-genutils==0.2.0
+ipywidgets==7.6.5
+jedi==0.18.0
+Jinja2==3.0.2
+jsonschema==4.0.1
+jupyter-client==7.0.6
+jupyter-core==4.8.1
+jupyterlab-pygments==0.1.2
+jupyterlab-widgets==1.0.2
+kiwisolver==1.3.2
+MarkupSafe==2.0.1
+matplotlib==3.4.3
+matplotlib-inline==0.1.3
+mistune==0.8.4
+nbclient==0.5.4
+nbconvert==6.2.0
+nbformat==5.1.3
+nest-asyncio==1.5.1
+notebook==6.4.4
 numpy==1.19.5
+packaging==21.0
 pandas==1.2.4
+pandocfilters==1.5.0
+parso==0.8.2
+patsy==0.5.2
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==8.3.2
 plotly==5.3.1
+prometheus-client==0.11.0
+prompt-toolkit==3.0.20
+protobuf==3.18.1
+ptyprocess==0.7.0
+pyarrow==5.0.0
+pycparser==2.20
+pydeck==0.7.0
+Pygments==2.10.0
+pyparsing==2.4.7
+pyrsistent==0.18.0
+python-dateutil==2.8.2
+pytz==2021.3
+pyzmq==22.3.0
+requests==2.26.0
+scipy==1.7.1
 seaborn==0.11.2
+Send2Trash==1.8.0
+six==1.16.0
+smmap==4.0.0
+statsmodels==0.13.0
 streamlit==0.89.0
-
-
+tenacity==8.0.1
+terminado==0.12.1
+testpath==0.5.0
+toml==0.10.2
+toolz==0.11.1
+tornado==6.1
+traitlets==5.1.0
+typing-extensions==3.10.0.2
+tzlocal==3.0
+urllib3==1.26.7
+validators==0.18.2
+watchdog==2.1.6
+wcwidth==0.2.5
+webencodings==0.5.1
+widgetsnbextension==3.5.1

stream_app.py

@@ -1,110 +1,55 @@
-from os import write
+# -*- coding: utf-8 -*-
 import streamlit as st
+
 import numpy as np
-from datetime import datetime
-import pandas as pd
 import plotly.express as px
-from plotly.subplots import make_subplots
-import plotly.graph_objects as go
-import matplotlib.pyplot as plt
-from streamlit.elements.arrow_altair import ChartType
-from streamlit.type_util import data_frame_to_bytes
-
-
-#The purpose of this script is to create interactive graphs
-#on an application through streamlit and plotly. These graphs will make it possible to
-#To answer certain questions about the link between the variables
-#Principle to see the distribution of fines according Define the variable that allows filtering to select boxes the variable that allows filtering to select boxes the types of violations, the countries of jurisdiction ...
-
-#Define the title of the application
-st.title('Data exploration & Visualization')
-
-#Description of the application
-st.markdown('This application allows you to create interactive charts with streamlit in order to answer a number of data-related questions and to better highlight the information contained in the dataset.')
-
-#Read the data from a cvs file
-df = pd.read_csv("theolex.csv")
-
-# Get the max year from date
-df.decision_date = pd.to_datetime(df.decision_date)
-max_year = df.decision_date.max().year
-
-
-#Define the variable that allows filtering to select boxes
-d = st.select_slider('Select year interval', options=range(max_year-10, max_year))
-
-# Plot a bar_chart graph to show the amount monetary sanction per organization
-df = df.set_index('name').sort_values('monetary_sanction_usd', ascending=False)
-st.header('Amount of monetary sanction per oranization name')
-
-
-st.bar_chart(df[df.decision_date.dt.year <= d].head(10)['monetary_sanction_usd'], height=500)
-st.markdown('From this graph we can find the companies that have the highest amount of monetary sanction for selected year.')
-
-
-#---------------------------------------------------------------------------------------
-# Plot a bar_chart graph to show the amount monetary sanction per nature of violation 
-
-st.header('Amount of monetary sanction per nature of violation')
-
-df = df.set_index('nature_of_violations').sort_values('monetary_sanction_usd', ascending=False)
-
-st.bar_chart(df[df.decision_date.dt.year <= d].head(10)['monetary_sanction_usd'], height=500)
-
-
-
-
-
-#----------------------------------------------------------------------------------------------------#
-# Plot a bar_chart graph to show the amount monetary sanction per country of the authorities
-
-df_filter = pd.read_csv("monetary_year.csv")
-df_filter.decision_date = pd.to_datetime(df_filter.decision_date)
-st.header('Amount of monetary sanction per country of the Authorities')
-
-df_filter = df_filter.set_index('country_authorities').sort_values('monetary_sanction', ascending=False)
-
-st.bar_chart(df_filter[df_filter.decision_date.dt.year <= d].head(10)['monetary_sanction'], height=500)
-#--------------------------------------------------------------------------------------------------
-
-#Here we want to see for a given country, how does the amount of monetary sanction changes per year
-
-d = st.selectbox('Select the country of the organisation', options=list(df_filter['country_organisation'].unique()))
-df_filter.decision_date = pd.to_datetime(df_filter.decision_date)
-#df_filter.year=df_filter.decision_date.dt.year
-
-df_filter = df_filter.set_index('year').sort_values('monetary_sanction', ascending=True)
-
-st.header('Amount of monetary sanction per year for a selected country of organisation')
-
-st.line_chart(df_filter[df_filter.country_organisation == d].head(10)['monetary_sanction'])
-st.markdown('From this grap shows the urve of the monetary sanction from 2010 to 2020 for eac contry. For example, if we check for US we can see that the amount monetary sanction increase from 2012 to to 2019 and decreases in 2019.')
-
-#-----------------------------------------------------------------------------------------------------
-
-
-
-# st.header('Amount of monetary sanction compare to the revenue of the company')
-
-# #df_filter = df_filter.sort_values('revenues_organisation', ascending=False)
-
-# df_filter = df_filter.set_index('revenues_organisation')
-
-# #st.area_chart(df_filter.head(10)['monetary_sanction'])
-
-# df_agg = df_filter.groupby('year').agg({'monetary_sanction': 'count'})
-# st.bar_chart(df_agg)
-
-
-#df_agg=df_filter.groupby('year').agg({'monetary_sanction':  'count'})
-st.header('The number of monetary sanction per country and per year')
-
-df_agg = pd.read_csv("agg_countMt.csv")
-d = st.select_slider('Select year interval', options=list(df_agg['year'].unique()))
-df_agg= df_agg.set_index('country_organisation')
-
-df_agg.columns=['year', 'num_monetary_sanction']
-st.bar_chart(df_agg[df_agg.year == d].head(10)['num_monetary_sanction'])
 
-#df_agg
-#ratio entre le revenue et le montant des sanction
+from data_processing import load_data, process_data
+
+st.title("Data Analysis 🌎 📃")
+st.write("by [Teolex](https://www.theolex.io/)")
+
+# load and process data
+data = load_data()
+decisions, organizations, authorities = process_data(data)
+
+st.sidebar.title("Parameters")
+authorities_country = st.sidebar.selectbox('Authority country', authorities.country.unique())
+
+select_auth = authorities[authorities.country == authorities_country].name.sort_values()
+authority = st.sidebar.selectbox('Authority', ['All', *select_auth])
+
+min_year, max_year = st.sidebar.slider('Decisions year', min_value=2001, max_value=2021, value=(2010, 2021))
+
+# apply filters
+authority_filter = True
+if authority != 'All':
+    authority_filter = decisions.authorities_name.apply(lambda a: authority in a)
+else:
+    authority_filter = decisions.authorities_name.apply(lambda a: bool(set(select_auth) & set(a)))
+
+
+decisions.head()
+year_filter = (decisions.year >= min_year) & (decisions.year <= max_year)
+
+decision_scope = decisions[authority_filter & year_filter]
+
+## explore monetary sanctions
+monetary_decision = decision_scope[decision_scope.monetary_sanction > 0]
+monetary_decision = monetary_decision[monetary_decision.org_revenues != ""]
+monetary_decision['org_revenues'] = monetary_decision.org_revenues.astype(float)
+monetary_decision['log10_org_revenues'] = monetary_decision.org_revenues.apply(np.log10)
+monetary_decision['log10_monetary_sanction'] = monetary_decision.monetary_sanction.apply(np.log10)
+monetary_decision['same_country'] = (monetary_decision.org_country == monetary_decision.authorities_country)
+fig = px.scatter(monetary_decision,
+                 x="org_revenues",
+                 y="monetary_sanction",
+                 log_x=True,
+                 log_y=True,template="simple_white",
+                 color="authorities_country",
+                 trendline="ols",
+                 hover_name="org_name")
+st.plotly_chart(fig)
+st.markdown("Comments...")
+st.subheader("1.1 Main takeaways: ")
+st.write("sample text.")