Merged 'main' into 'pquiggles/improvements' accepting 'main's changes

src/notebooks/classifiers/work_experience/investment_banking_group_classifier.ipynb

@@ -9,7 +9,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/Users/paulquigley/ZSL_projects/VSP-data-enrichment/src\n"
+      "/Volumes/code/geometric/VSP-data-enrichment/src\n"
      ]
     }
    ],
@@ -39,31 +39,45 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[2m2024-09-16 15:16:21\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:21\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:21\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:21\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:49\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:49\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:49\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:49\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:50\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:50\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:50\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:50\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:50\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:50\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:51\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:51\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1125\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m96\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:51\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:51\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1052\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m104\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:52\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:52\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1052\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m116\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:52\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:52\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1051\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m108\u001b[0m\n",
       "{\n",
       "  \"investment_banking_group\": \"Private Funds\",\n",
       "  \"confidence\": 0.9,\n",
       "  \"reasoning\": \"The candidate's role as an Investment Professional at Accel-KKR, a private equity firm focused on technology investments, aligns closely with the PRIVATE_FUNDS group. Their work involves acquisitions, buyouts, and structured investments, which are typical activities in private equity. Additionally, the candidate's experience at Fidelity Investments in corporate finance suggests a strong background in financial transactions, further supporting this classification.\"\n",
       "}\n",
       "{\n",
-      "  \"investment_banking_group\": \"Generalist\",\n",
+      "  \"investment_banking_group\": \"Financial Institutions Group (FIG) / Fintech\",\n",
       "  \"confidence\": 0.8,\n",
-      "  \"reasoning\": \"The candidate worked in the Investment Banking division at William Blair & Company, a firm known for providing a wide range of financial advisory services, which suggests a generalist role. The absence of specific details in the work experience description indicates that the candidate may have been involved in various transactions across different sectors rather than specializing in a particular area. Additionally, their subsequent role at Accel-KKR, a private equity firm focused on technology, further supports the idea of a generalist background prior to specializing.\"\n",
+      "  \"reasoning\": \"The candidate worked in the FFAS Corporate Finance division at Fidelity Investments, a major player in the financial services sector. This role likely involved financial advisory services related to financial institutions, aligning with the focus of the FINANCIAL_INSTITUTIONS group. The candidate's experience at Fidelity, combined with their education in Business Economics, supports this classification, although the lack of specific details in the job description leaves some uncertainty.\"\n",
       "}\n",
       "{\n",
-      "  \"investment_banking_group\": \"Financial Institutions Group (FIG) / Fintech\",\n",
+      "  \"investment_banking_group\": \"Generalist\",\n",
       "  \"confidence\": 0.8,\n",
-      "  \"reasoning\": \"The candidate worked in the FFAS Corporate Finance division at Fidelity Investments, a major player in the financial services sector. This role likely involved financial advisory services related to financial institutions, aligning with the focus of the FINANCIAL_INSTITUTIONS group. The candidate's experience at Fidelity, combined with their education in Business Economics, supports this classification, although the lack of specific details in the job description leaves some uncertainty.\"\n",
+      "  \"reasoning\": \"The candidate worked in the Investment Banking division at William Blair & Company, a firm known for providing a wide range of financial advisory services, which suggests a generalist role. The absence of specific details in the work experience description indicates that the candidate may have been involved in various transactions across different sectors rather than specializing in a particular area. Additionally, their subsequent role at Accel-KKR, a private equity firm focused on technology, further supports the idea of a generalist background prior to specializing.\"\n",
       "}\n",
       "{\n",
       "  \"investment_banking_group\": \"Financial Institutions Group (FIG) / Fintech\",\n",

src/notebooks/classifiers/work_experience/investment_focus_asset_class_classifier.ipynb

@@ -2,14 +2,14 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/Users/paulquigley/ZSL_projects/VSP-data-enrichment/src\n"
+      "/Volumes/code/geometric/VSP-data-enrichment/src\n"
      ]
     }
    ],
@@ -39,31 +39,31 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[2m2024-09-16 15:14:55\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:55\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:55\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1085\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m93\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1084\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m96\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1085\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m99\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:14:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1158\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m114\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:44\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:44\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:44\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:44\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:45\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:45\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:45\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:45\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:45\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:45\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:46\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:46\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1085\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m93\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:46\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:46\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1084\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m96\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:46\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:46\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1085\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m99\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:47\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:47\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1158\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m114\u001b[0m\n",
       "{\n",
       "  \"investing_focus_asset_class\": \"Unclassifiable\",\n",
       "  \"confidence\": 0.3,\n",

src/notebooks/classifiers/work_experience/investment_focus_sectors_classifier.ipynb

@@ -9,7 +9,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/Users/paulquigley/ZSL_projects/VSP-data-enrichment/src\n"
+      "/Volumes/code/geometric/VSP-data-enrichment/src\n"
      ]
     }
    ],
@@ -46,33 +46,33 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[2m2024-09-16 15:13:36\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:36\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:36\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:37\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:37\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:37\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:37\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:37\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:38\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:38\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:40\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:40\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1081\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m88\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:40\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:40\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1080\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m93\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:40\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:40\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1081\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m94\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:40\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
-      "\u001b[2m2024-09-16 15:13:40\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1154\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m96\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:30\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:30\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:30\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:30\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:30\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:30\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:30\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:30\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:31\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:31\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:32\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:32\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1081\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m87\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:32\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:32\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1080\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m88\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:32\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:32\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1081\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m94\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:32\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:21:32\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1154\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m96\u001b[0m\n",
       "{\n",
       "  \"investing_focus_sector\": \"Financial Institutions Group (FIG) / Fintech\",\n",
       "  \"confidence\": 0.7,\n",
-      "  \"reasoning\": \"The specific work experience item indicates that the candidate worked at Fidelity Investments, a company clearly categorized under financial services. The title \\\"FFAS Corporate Finance\\\" suggests involvement in financial analysis and corporate finance activities, which aligns with the financial institutions sector. However, the lack of detailed responsibilities in the description leads to a slightly lower confidence level.\"\n",
+      "  \"reasoning\": \"The specific work experience item is at Fidelity Investments, a company clearly identified within the financial services sector. The title \\\"FFAS Corporate Finance\\\" suggests a focus on financial analysis and corporate finance activities, which aligns with the financial institutions category. However, the lack of detailed responsibilities in the description leads to a slightly lower confidence level.\"\n",
       "}\n",
       "{\n",
       "  \"investing_focus_sector\": \"Financial Institutions Group (FIG) / Fintech\",\n",
       "  \"confidence\": 0.8,\n",
-      "  \"reasoning\": \"The specific work experience item indicates that the candidate worked at Fidelity Investments, a company clearly categorized under financial services. The title \\\"PI Corporate Finance\\\" suggests involvement in financial activities, likely related to investment management or corporate finance within the financial sector. While the description lacks detail, the context of the company and the role strongly supports classification in the financial institutions sector.\"\n",
+      "  \"reasoning\": \"The specific work experience item indicates that the candidate worked at Fidelity Investments, a company clearly categorized under financial services. The title \\\"PI Corporate Finance\\\" suggests involvement in financial activities related to private investments, which aligns with the financial institutions sector. While the description lacks detail, the company name and industry classification provide strong evidence for this classification.\"\n",
       "}\n",
       "{\n",
       "  \"investing_focus_sector\": \"Financial Institutions Group (FIG) / Fintech\",\n",

src/notebooks/classifiers/work_experience/work_experience_classifier.ipynb

@@ -2,14 +2,14 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "/Users/paulquigley/ZSL_projects/VSP-data-enrichment/src\n"
+      "/Volumes/code/geometric/VSP-data-enrichment/src\n"
      ]
     }
    ],
@@ -39,82 +39,102 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
-      "\u001b[2m2024-09-16 15:16:55\u001b[0m [\u001b[32m\u001b[1mdebug    \u001b[0m] \u001b[1mLLM cache hit                 \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:55\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:55\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:55\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/openai/api_key\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mFetching parameter from Parameter Store\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mCreating boto3 client         \u001b[0m \u001b[36mservice\u001b[0m=\u001b[35mssm\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:56\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mAssuming role                 \u001b[0m \u001b[36mrole_arn\u001b[0m=\u001b[35marn:aws:iam::339713096219:role/ReadWrite\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mRole assumed successfully     \u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:57\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mSuccessfully fetched parameter\u001b[0m \u001b[36mparameter\u001b[0m=\u001b[35m/secrets/rapidapi/linkedin\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:58\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:58\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1811\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m122\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:58\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:58\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m2005\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m134\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:58\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:58\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1816\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m142\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:58\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:58\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1816\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m139\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:59\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:59\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1811\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m142\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:59\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:59\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1812\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m147\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:59\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:19:59\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1847\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m119\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:20:00\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:20:00\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1813\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m155\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:20:00\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:20:00\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1814\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m165\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:20:00\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mOpenAI API called             \u001b[0m \u001b[36mmodel\u001b[0m=\u001b[35mgpt-4o-mini\u001b[0m\n",
+      "\u001b[2m2024-09-16 15:20:00\u001b[0m [\u001b[32m\u001b[1minfo     \u001b[0m] \u001b[1mToken usage                   \u001b[0m \u001b[36minput_tokens\u001b[0m=\u001b[35m1815\u001b[0m \u001b[36moutput_tokens\u001b[0m=\u001b[35m182\u001b[0m\n",
       "{\n",
       "  \"primary_job_type\": \"Full-time\",\n",
       "  \"secondary_job_type\": \"Investing\",\n",
       "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience as a Partner at HOF Capital is a full-time role in a venture capital firm, which aligns with the candidate's ongoing career in investing. Given that the candidate has been involved in venture capital and private equity roles, this position is classified as FULL_TIME. The secondary job type is classified as INVESTING, as the role directly involves investment activities typical of a venture capital firm. This classification is supported by the candidate's previous experience as Vice President at Bessemer Venture Partners and their role as Co-Founder at The MBA Fund, both of which are also in the investing domain.\"\n",
+      "  \"reasoning\": \"The work experience as a Partner at HOF Capital is a full-time role in a venture capital firm, which aligns with the candidate's ongoing career in investing. Given that the candidate has been involved in venture capital and private equity roles, this position is classified as FULL_TIME. The secondary job type is classified as INVESTING, as the role directly involves investment activities typical of a venture capital firm. The candidate's previous experience as Vice President at Bessemer Venture Partners further supports this classification.\"\n",
       "}\n",
       "{\n",
       "  \"primary_job_type\": \"Full-time\",\n",
       "  \"secondary_job_type\": \"Investing\",\n",
       "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience at The MBA Fund is classified as FULL_TIME because it is a co-founding role in a venture capital firm, indicating ongoing and regular employment. The responsibilities described involve significant engagement with startups and investment activities, which aligns with a full-time commitment. For the secondary job type, it is classified as INVESTING since the role involves backing and supporting startup founders, which is characteristic of venture capital activities. This is supported by the description of the firm and the candidate's role as a General Partner, which typically involves making investment decisions and managing a portfolio of investments.\"\n",
+      "  \"reasoning\": \"The work experience item indicates that the candidate is a Co-Founder and General Partner at The MBA Fund, which is a venture capital firm. This role is ongoing and involves significant responsibilities, indicating it is a full-time position rather than an internship or part-time role. Therefore, it is classified as FULL_TIME. Additionally, since the role is directly related to venture capital and involves investing in startups, it fits the INVESTING category for the secondary job type. The candidate's extensive background in venture capital, as seen in their other roles, supports this classification.\"\n",
       "}\n",
       "{\n",
-      "  \"primary_job_type\": \"Full-time\",\n",
+      "  \"primary_job_type\": \"Other\",\n",
       "  \"secondary_job_type\": \"Investing\",\n",
-      "  \"confidence\": 1.0,\n",
-      "  \"reasoning\": \"The work experience as Vice President at Bessemer Venture Partners is classified as FULL_TIME because it is a full-time position that spans over four years, which aligns with the criteria for regular ongoing employment. The secondary job type is classified as INVESTING, as the role is within a venture capital firm, which involves professional investing activities. This is supported by the candidate's extensive background in venture capital and private equity, as indicated in both the resume and the specific work experience item.\"\n",
+      "  \"confidence\": 0.9,\n",
+      "  \"reasoning\": \"The work experience as a Board Observer at Rillavoice is a role that typically involves providing guidance and oversight to the company's management team, which aligns with advisory responsibilities rather than a full-time employment setup. Given that this position is part of a broader involvement in venture capital and investment activities, it does not fit the criteria for a full-time job. Therefore, it is classified as OTHER for the primary job type. For the secondary job type, since the role involves oversight and advisory functions in a startup context, it aligns with the INVESTING category, as it is related to venture capital activities.\"\n",
       "}\n",
       "{\n",
-      "  \"primary_job_type\": \"Other\",\n",
-      "  \"secondary_job_type\": \"Other\",\n",
+      "  \"primary_job_type\": \"Full-time\",\n",
+      "  \"secondary_job_type\": \"CorpDev / Strategy\",\n",
       "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience as a Board Observer at Rillavoice is a role that typically involves providing guidance and oversight to the company's management team, which aligns with advisory responsibilities. Given that this position is not a full-time employment role and does not fit the criteria for an internship, it is classified as OTHER for the primary job type. For the secondary job type, since the role involves oversight and strategic input in a software company, it does not fit into the categories of investing, banking, or consulting, thus it is classified as OTHER as well.\"\n",
+      "  \"reasoning\": \"The work experience as a Board Director at Luxury Presence is a long-term role from 2021 to 2024, indicating a significant commitment that aligns with a full-time position rather than a temporary or part-time role. Given that the candidate is involved in a board capacity, this suggests a leadership and strategic role rather than an advisory or internship position. For the secondary job type, the role of Board Director typically involves oversight and strategic decision-making, which aligns with CORPDEV_STRATEGY as it relates to corporate governance and strategic planning within the company.\"\n",
       "}\n",
       "{\n",
-      "  \"primary_job_type\": \"Other\",\n",
+      "  \"primary_job_type\": \"Full-time\",\n",
       "  \"secondary_job_type\": \"Investing\",\n",
       "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience at Archy is classified as an INVESTING role because the title \\\"Investor\\\" indicates a professional investing position, which aligns with the candidate's experience in venture capital as seen in their role as Vice President at Bessemer Venture Partners and Co-Founder at The MBA Fund. The duration from 2022 to 2024 suggests a longer-term commitment, which is typical for investing roles rather than internships. The primary job type is classified as OTHER since there is no indication of full-time employment or internship status, and it does not fit into the other primary categories.\"\n",
+      "  \"reasoning\": \"The work experience as an \\\"Investor\\\" at Spot AI from 2021 to 2024 indicates a professional investing role, which aligns with the secondary job type category of INVESTING. The candidate's full resume shows a strong background in venture capital and investment roles, including being a Vice President at Bessemer Venture Partners and a Co-Founder at The MBA Fund, both of which further support the classification as INVESTING. Since the role spans over three years and does not fit the criteria for an internship or other primary job types, it is classified as FULL_TIME for the primary job type.\"\n",
       "}\n",
       "{\n",
-      "  \"primary_job_type\": \"Advisory / Board / Independent Investor\",\n",
-      "  \"secondary_job_type\": \"Other\",\n",
+      "  \"primary_job_type\": \"Other\",\n",
+      "  \"secondary_job_type\": \"Investing\",\n",
       "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience as a Board Observer at MaintainX is a role that typically involves providing guidance and oversight to the company's management team, which aligns with advisory responsibilities. Given that this position is not a full-time role and does not indicate a regular employment setup, it does not fit the FULL_TIME category. The duration from 2021 to 2024 suggests a longer-term commitment, but the nature of the role indicates it is more advisory in nature. Therefore, I classify it as ADVISORY_BOARD_INVESTOR for the primary job type. For the secondary job type, since the role involves oversight and strategic input rather than direct investment activities, it does not fit into the INVESTING category. Instead, it aligns more closely with the OTHER category, as it does not fit the other defined roles.\"\n",
+      "  \"reasoning\": \"The work experience at Archy is classified as an INVESTING role because the title \\\"Investor\\\" indicates a professional investing position, which aligns with the candidate's experience in venture capital as seen in their role as Vice President at Bessemer Venture Partners and Co-Founder at The MBA Fund. The duration from 2022 to 2024 suggests a longer-term commitment, which is typical for investing roles rather than internships. The primary job type is classified as OTHER since there is no indication of regular employment or a formal structure like full-time or part-time; it appears to be more of an investment role without a traditional employment setup.\"\n",
       "}\n",
       "{\n",
       "  \"primary_job_type\": \"Full-time\",\n",
       "  \"secondary_job_type\": \"Investing\",\n",
-      "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience at Spot AI is classified as an INVESTING role because the title \\\"Investor\\\" indicates a professional investing position, which aligns with the candidate's experience in venture capital and investment roles as seen in their resume. The duration from 2021 to 2024 suggests a long-term commitment, which rules out the possibility of it being an internship. The primary job type is classified as FULL_TIME since the role is ongoing and likely compensated, fitting the criteria for regular employment.\"\n",
+      "  \"confidence\": 1.0,\n",
+      "  \"reasoning\": \"The work experience as Vice President at Bessemer Venture Partners is classified as FULL_TIME because it is a full-time position that spans over four years, which aligns with the criteria for regular ongoing employment. The secondary job type is classified as INVESTING, as the role is within a venture capital firm, which involves professional investing activities. This is supported by the candidate's extensive background in venture capital and private equity, as indicated in both the resume and the specific work experience item.\"\n",
       "}\n",
       "{\n",
       "  \"primary_job_type\": \"Other\",\n",
-      "  \"secondary_job_type\": \"Entrepreneur / Founder\",\n",
+      "  \"secondary_job_type\": \"Other\",\n",
       "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience as a Board Observer at VendorPM is a role that typically involves providing guidance and oversight to the company's management team, which aligns with advisory responsibilities rather than a full-time employment setup. Given that the candidate has been involved in this role from 2021 to 2024, it suggests a part-time or advisory capacity rather than a full-time position. Therefore, the Primary Job Type is classified as OTHER, as it does not fit the other categories. For the Secondary Job Type, this role is best classified as ENTREPRENEUR_FOUNDER since it involves oversight and advisory functions in a startup environment, which is common for board observer roles in venture-backed companies.\"\n",
+      "  \"reasoning\": \"The work experience as a Board Observer at VendorPM is a role that typically involves providing guidance and oversight to the company's management team, which aligns with advisory responsibilities rather than a full-time employment setup. Given that the candidate has been in this role from 2021 to 2024, it suggests a part-time or advisory capacity rather than a full-time position. Therefore, the Primary Job Type is classified as OTHER, as it does not fit the other categories. For the Secondary Job Type, the role of Board Observer does not involve direct investment activities but rather oversight, which does not fit into the INVESTING category. Thus, it is classified as OTHER as well.\"\n",
       "}\n",
       "{\n",
-      "  \"primary_job_type\": \"Full-time\",\n",
+      "  \"primary_job_type\": \"Other\",\n",
       "  \"secondary_job_type\": \"CorpDev / Strategy\",\n",
       "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience as a Board Director at Luxury Presence is a long-term role from 2021 to 2024, indicating a significant commitment that aligns with a full-time position rather than a temporary or part-time role. Given that this position involves governance and oversight, it does not fit the criteria for an internship or extracurricular activity. In terms of secondary job type, the role of Board Director typically involves strategic decision-making and oversight, which aligns with CORPDEV_STRATEGY as it is a corporate governance role rather than a direct investment role.\"\n",
+      "  \"reasoning\": \"The work experience as a Board Observer at MaintainX is a role that typically involves providing guidance and oversight to the company's management team, which aligns with advisory responsibilities. Given that this position is not a full-time role and does not indicate a regular employment setup, it does not fit the FULL_TIME category. The duration from 2021 to 2024 suggests a longer-term commitment, but it is not a traditional employment role, thus it is classified as OTHER for the primary job type. For the secondary job type, as a Board Observer, the role involves strategic oversight rather than direct investment activities, which aligns it more closely with the CORPDEV_STRATEGY category rather than INVESTING.\"\n",
       "}\n",
       "{\n",
       "  \"primary_job_type\": \"Other\",\n",
       "  \"secondary_job_type\": \"Investing\",\n",
       "  \"confidence\": 0.9,\n",
-      "  \"reasoning\": \"The work experience at ServiceTitan is classified as an INVESTING role because the title \\\"Investor\\\" indicates a focus on investment activities, which aligns with the candidate's background in venture capital and investment roles as seen in their resume. The candidate has been involved in various investment capacities, including their role as Vice President at Bessemer Venture Partners and as a Co-Founder at The MBA Fund, both of which emphasize their expertise in investing. The duration of the role from 2021 to 2024 suggests a longer-term commitment, which is typical for professional investing roles rather than internships or part-time positions. For the Primary Job Type, since this role is not a full-time position but rather an investment role, it is classified as OTHER, as it does not fit the other primary categories like FULL_TIME or INTERNSHIP.\"\n",
+      "  \"reasoning\": \"The work experience at ServiceTitan is classified as an INVESTING role because the title \\\"Investor\\\" indicates a focus on investment activities, which aligns with the candidate's background in venture capital and investment roles as seen in their resume. The candidate has been involved in various investment capacities, including their role as Vice President at Bessemer Venture Partners and as a Co-Founder at The MBA Fund, both of which emphasize their expertise in investing. The duration of the role from 2021 to 2024 suggests a longer-term commitment, which is typical for professional investing roles rather than internships or part-time positions. For the Primary Job Type, since this role is not a full-time ongoing employment position but rather an investment role, it is classified as OTHER, as it does not fit the other primary categories.\"\n",
       "}\n"
      ]
     }

src/vsp/app/main.py

@@ -17,18 +17,12 @@ Usage:
 """
 
 import asyncio
-from typing import Sequence, Mapping, List
-
-import calendar
-from datetime import date
-
+from typing import Sequence
 
 from pydantic import BaseModel, Field
 
 from vsp.app.classifiers.education_classifier import EducationClassification, EducationClassifier
 from vsp.app.classifiers.work_experience.general_work_experience_classifier import (
-    PrimaryJobType,
-    SecondaryJobType,
     WorkExperienceClassification,
     WorkExperienceClassifier,
 )
@@ -47,8 +41,6 @@ from vsp.app.classifiers.work_experience.investment_banking_group_classifier imp
 from vsp.app.model.linkedin.linkedin_models import Education, LinkedinProfile, Position
 
 
-from collections import defaultdict
-
 class ClassifiedEducation(BaseModel):
     """
     Represents a classified education item from a LinkedIn profile.
@@ -68,10 +60,17 @@ class ClassifiedWorkExperience(BaseModel):
 
     Attributes:
         position (Position): The original position item from the LinkedIn profile.
-        work_experience_classification (WorkExperienceClassification): The general classification results for the work experience.
-        investment_banking_classification (InvestmentBankingGroupClassification | None): The investment banking classification results, if applicable.
-        investing_focus_asset_class_classification (InvestingFocusAssetClassClassification | None): The investing focus asset class classification results, if applicable.
-        investing_focus_sector_classification (InvestingFocusSectorClassification | None): The investing focus sector classification results, if applicable.
+        work_experience_classification (WorkExperienceClassification): The general classification results
+        for the work experience.
+        investment_banking_classification (
+            InvestmentBankingGroupClassification | None
+        ): The investment banking classification results, if applicable.
+        investing_focus_asset_class_classification (
+            InvestingFocusAssetClassClassification | None
+        ): The investing focus asset class classification results, if applicable.
+        investing_focus_sector_classification (
+            InvestingFocusSectorClassification | None
+        ): The investing focus sector classification results, if applicable.
     """
 
     position: Position
@@ -87,13 +86,12 @@ class LinkedinProfileClassificationResults(BaseModel):
 
     Attributes:
         classified_educations (Sequence[ClassifiedEducation]): A sequence of classified education items.
-        classified_work_experiences (Sequence[ClassifiedWorkExperience]): A sequence of classified work experience items.
+        classified_work_experiences (Sequence[ClassifiedWorkExperience]): A sequence of classified work
+        experience items.
     """
 
     classified_educations: Sequence[ClassifiedEducation] = Field(default_factory=list)
     classified_work_experiences: Sequence[ClassifiedWorkExperience] = Field(default_factory=list)
-    full_time_work_experience_years: float = Field(default=0.0)
-    full_time_work_experience_by_secondary: Mapping[SecondaryJobType, float] = Field(default_factory=dict)
 
 
 class VspDataEnrichment:
@@ -107,11 +105,13 @@ class VspDataEnrichment:
         education_classifier (EducationClassifier): Classifier for education items.
         work_experience_classifier (WorkExperienceClassifier): Classifier for general work experiences.
         investment_banking_classifier (InvestmentBankingGroupClassifier): Classifier for investment banking groups.
-        investing_focus_asset_class_classifier (InvestingFocusAssetClassClassifier): Classifier for investing focus asset classes.
+        investing_focus_asset_class_classifier (
+            InvestingFocusAssetClassClassifier
+        ): Classifier for investing focus asset classes.
         investing_focus_sector_classifier (InvestingFocusSectorClassifier): Classifier for investing focus sectors.
     """
 
-    def __init__(self):
+    def __init__(self) -> None:
         """Initialize the VspDataEnrichment class with all required classifiers."""
         self._education_classifier = EducationClassifier()
         self._work_experience_classifier = WorkExperienceClassifier()
@@ -119,83 +119,6 @@ class VspDataEnrichment:
         self._investing_focus_asset_class_classifier = InvestingFocusAssetClassClassifier()
         self._investing_focus_sector_classifier = InvestingFocusSectorClassifier()
 
-
-    def estimate_full_time_experience_by_secondary_job_type(self, classified_work_experiences: List[ClassifiedWorkExperience]) -> Mapping[SecondaryJobType, float]:
-        # Define current date
-        current_date = date(2024, 9, 18)
-        
-        # List to store all events (start or end of intervals)
-        events = []
-        
-        # Set to store all observed SecondaryJobTypes
-        observed_secondary_job_types = set()
-        
-        for cwe in classified_work_experiences:
-            classification = cwe.work_experience_classification.primary_job_type
-            secondary_job_type = cwe.work_experience_classification.secondary_job_type
-            
-            if classification == PrimaryJobType.FULL_TIME and secondary_job_type:
-                # Normalize start date
-                start = cwe.position.start
-                if not start or not start.year:
-                    continue
-                start_year = start.year
-                start_month = start.month if start.month else 1
-                start_day = start.day if start.day else 1
-                start_date = date(start_year, start_month, start_day)
-                
-                # Normalize end date
-                end = cwe.position.end
-                if end is None:
-                    end_date = current_date
-                else:
-                    if not end.year:
-                        continue
-                    end_year = end.year
-                    end_month = end.month if end.month else 12
-                    if end.day:
-                        end_day = end.day
-                    else:
-                        # Get last day of the month
-                        _, end_day = calendar.monthrange(end_year, end_month)
-                    end_date = date(end_year, end_month, end_day)
-                
-                if start_date > end_date:
-                    continue  # Skip invalid intervals
-                
-                # Add events for sweep-line algorithm
-                events.append((start_date, 'start', secondary_job_type))
-                events.append((end_date, 'end', secondary_job_type))
-                
-                observed_secondary_job_types.add(secondary_job_type)
-        
-        # Sort events by date
-        events.sort(key=lambda x: x[0])
-        
-        active_secondary_job_types = set()
-        last_date = None
-        durations = defaultdict(int)  # in days
-        
-        for event_date, event_type, secondary_job_type in events:
-            if last_date is not None and event_date > last_date:
-                interval_duration = (event_date - last_date).days
-                # Distribute the interval_duration among active_secondary_job_types
-                for active_type in active_secondary_job_types:
-                    durations[active_type] += interval_duration
-            
-            if event_type == 'start':
-                active_secondary_job_types.add(secondary_job_type)
-            elif event_type == 'end':
-                active_secondary_job_types.discard(secondary_job_type)
-            
-            last_date = event_date
-        
-        # Convert durations from days to years
-        durations_in_years = {stype: round(days / 365.25, 2) for stype, days in durations.items()}
-        
-        return durations_in_years
-
-
     async def process_linkedin_profile(self, profile: LinkedinProfile) -> LinkedinProfileClassificationResults:
         """
         Process a LinkedIn profile and classify its education and work experiences.
@@ -242,14 +165,13 @@ class VspDataEnrichment:
                 work_classification.primary_job_type.INTERNSHIP,
                 work_classification.primary_job_type.EXTRACURRICULAR,
             }:
-    
                 if work_classification.secondary_job_type == work_classification.secondary_job_type.INVESTMENT_BANKING:
                     ib_classification = await self._investment_banking_classifier.classify_investment_banking_group(
                         profile, position
                     )
                     classified_work_experience.investment_banking_classification = ib_classification
 
-                if work_classification.secondary_job_type == work_classification.secondary_job_type.INVESTING and not work_classification.primary_job_type == work_classification.primary_job_type.ADVISORY_BOARD_INVESTOR:
+                if work_classification.secondary_job_type == work_classification.secondary_job_type.INVESTING:
                     asset_class_task = (
                         self._investing_focus_asset_class_classifier.classify_investing_focus_asset_class(
                             profile, position
@@ -266,11 +188,8 @@ class VspDataEnrichment:
 
             classified_work_experiences.append(classified_work_experience)
 
-        
-        experience_by_job_type = self.estimate_full_time_experience_by_secondary_job_type(classified_work_experiences)
-        total_work_experience = sum(experience_by_job_type.values())
         return LinkedinProfileClassificationResults(
-            classified_educations=classified_educations, classified_work_experiences=classified_work_experiences, full_time_work_experience_years=total_work_experience, full_time_work_experience_by_secondary=experience_by_job_type
+            classified_educations=classified_educations, classified_work_experiences=classified_work_experiences
         )
 
 

tests/test_harness/enum_classifier_test.py

@@ -5,7 +5,7 @@ from typing import Any
 
 from pydantic import BaseModel
 
-from vsp.app.main import process_linkedin_profile
+from vsp.app.main import VspDataEnrichment
 from vsp.app.model.linkedin.linkedin_models import LinkedinProfile
 from vsp.app.model.vsp.vsp_models import VSPProfile
 from vsp.shared import logger_factory
@@ -45,7 +45,8 @@ def load_profiles() -> dict[str, tuple[LinkedinProfile, VSPProfile]]:
 
 
 async def compare_profiles(linkedin_profile: LinkedinProfile, classified_profile: VSPProfile) -> ComparisonResult:
-    result = await process_linkedin_profile(linkedin_profile)
+    vsp_enrichment = VspDataEnrichment()
+    result = await vsp_enrichment.process_linkedin_profile(linkedin_profile)
     comparisons = {}
     correct_enums = 0
     total_enums = 0