fix the requests to the SEC API

.env.example

@@ -9,4 +9,5 @@ WEBHOOK_SECRET=
 SPACE_URL=
 HF_TOKEN=
 HF_DATASET_REPO=
-FMP_API_TOKEN=
+FMP_API_TOKEN=
+SEC_API_TOKEN=

requirements.txt

@@ -33,3 +33,5 @@ typing-extensions>=4.5.0
 pytz>=2023.3
 
 TA-Lib>=0.4.19
+
+sec-api

src/api/insiders/insider_trading_aggregator.py

@@ -109,6 +109,67 @@ class InsiderTradingAggregator:
             # This was a major performance bottleneck.
             # await asyncio.sleep(60 / self.apis[api_name]['requests_per_minute'])
 
+    '''
+    async def _make_post_request(self, session: aiohttp.ClientSession, api_name: str,
+                             endpoint: str, payload: Dict) -> Optional[Dict]:
+    """Make an async rate-limited POST request to an API."""
+    if not self._check_rate_limit(api_name):
+        logger.warning(f"Daily rate limit reached for {api_name}")
+        return None
+    
+    if not self.apis[api_name]['api_key']:
+        logger.warning(f"No API key set for {api_name}, skipping request.")
+        return None
+
+    async with self.semaphores[api_name]:
+        try:
+            url = f"{self.apis[api_name]['base_url']}/{endpoint}"
+            
+            # Set Authorization header per SEC API docs (no "Bearer" prefix)
+            headers = {
+                'Authorization': self.apis[api_name]['api_key'],
+                'Content-Type': 'application/json'
+            }
+            
+            async with session.post(url, json=payload, headers=headers) as response:
+                response.raise_for_status()
+                self.request_counts[api_name] += 1
+                return await response.json()
+
+        except aiohttp.ClientError as e:
+            logger.error(f"POST request failed for {api_name} ({url}): {e}")
+            return None
+    '''
+
+    async def _make_post_request(self, session: aiohttp.ClientSession, api_name: str,
+                                 endpoint: str, payload: Dict) -> Optional[Dict]:
+        """Make an async rate-limited POST request to an API."""
+        if not self._check_rate_limit(api_name):
+            logger.warning(f"Daily rate limit reached for {api_name}")
+            return None
+
+        if not self.apis[api_name]['api_key']:
+            logger.warning(f"No API key set for {api_name}, skipping request.")
+            return None
+
+        async with self.semaphores[api_name]:
+            try:
+                # Build the full URL
+                url = f"{self.apis[api_name]['base_url']}/{endpoint}"
+
+                # Add API key as query parameter (SEC API supports this method)
+                url_with_token = f"{url}?token={self.apis[api_name]['api_key']}"
+
+                # Make POST request with JSON payload
+                async with session.post(url_with_token, json=payload) as response:
+                    response.raise_for_status()
+                    self.request_counts[api_name] += 1
+                    return await response.json()
+
+            except aiohttp.ClientError as e:
+                logger.error(f"POST request failed for {api_name} ({url}): {e}")
+                return None
+
     async def get_fmp_insider_trades(self, session: aiohttp.ClientSession,
                                      symbol: str, limit: int = 100, filter_days: int = 30) -> List[InsiderTrade]:
         """
@@ -182,6 +243,7 @@ class InsiderTradingAggregator:
             f"Retrieved {len(all_trades)} trades from FMP for {symbol} by iterating through {len(date_range)} days.")
         return all_trades
 
+    '''
     async def get_sec_api_insider_trades(self, session: aiohttp.ClientSession,
                                          symbol: str, limit: int = 100) -> List[InsiderTrade]:
         """Get insider trades from SEC-API"""
@@ -215,7 +277,185 @@ class InsiderTradingAggregator:
                 continue
 
         logger.info(f"Retrieved {len(trades)} trades from SEC-API for {symbol}")
-        return trades
+        return trades 
+
+
+    async def get_sec_api_insider_trades(self, session: aiohttp.ClientSession,
+                                         symbol: str, limit: int = 50,
+                                         filter_days: int = 30) -> List[InsiderTrade]:
+        """Get insider trades from SEC-API using a POST request with proper query structure."""
+        all_trades = []
+
+        # Ensure the limit does not exceed the API's maximum
+        if limit > 50:
+            limit = 50
+
+        # Date range for filtering
+        to_date = datetime.now()
+        from_date = to_date - timedelta(days=filter_days)
+
+        start_index = 0
+
+        while True:
+            # Construct query payload according to SEC API documentation
+            query_payload = {
+                "query": {
+                    "query_string": {
+                        "query": f'issuer.tradingSymbol:"{symbol}" AND periodOfReport:[{from_date.strftime("%Y-%m-%d")} TO {to_date.strftime("%Y-%m-%d")}]'
+                    }
+                },
+                "from": str(start_index),
+                "size": str(limit),
+                "sort": [{"filedAt": {"order": "desc"}}]
+            }
+
+            endpoint = "insider-trading"
+            logger.info(f"Requesting SEC-API data for {symbol}, starting at index {start_index}...")
+            data = await self._make_post_request(session, 'sec_api', endpoint, query_payload)
+
+            if not data or 'transactions' not in data or not data['transactions']:
+                break
+
+            for filing in data['transactions']:
+                # Parse both non-derivative and derivative transactions
+                transactions_list = filing.get('nonDerivativeTable', {}).get('transactions', []) + \
+                                    filing.get('derivativeTable', {}).get('transactions', [])
+
+                for transaction in transactions_list:
+                    try:
+                        # Map transaction codes to buy/sell
+                        trans_code = transaction.get('coding', {}).get('code', '')
+                        if trans_code in ('A', 'P'):  # Acquisition or Purchase
+                            trans_type = TransactionType.BUY.value
+                        elif trans_code in ('D', 'S'):  # Disposition or Sale
+                            trans_type = TransactionType.SELL.value
+                        else:
+                            continue  # Skip other transaction types
+
+                        shares = transaction.get('amounts', {}).get('shares')
+                        price_per_share = transaction.get('amounts', {}).get('pricePerShare')
+
+                        if shares is None or price_per_share is None:
+                            continue
+
+                        shares = int(shares)
+                        price_per_share = float(price_per_share)
+
+                        insider_trade = InsiderTrade(
+                            symbol=filing.get('issuer', {}).get('tradingSymbol', symbol),
+                            company_name=filing.get('issuer', {}).get('name', ''),
+                            insider_name=filing.get('reportingOwner', {}).get('name', ''),
+                            position=filing.get('reportingOwner', {}).get('relationship', {}).get('officerTitle',
+                                                                                                  'Insider'),
+                            transaction_date=transaction.get('transactionDate', {}).get('value', ''),
+                            transaction_type=trans_type,
+                            shares=shares,
+                            price=price_per_share,
+                            value=float(shares * price_per_share),
+                            form_type=filing.get('documentType', ''),
+                            source='SEC-API',
+                            filing_date=filing.get('filedAt', '')
+                        )
+                        all_trades.append(insider_trade)
+
+                    except (ValueError, TypeError, AttributeError) as e:
+                        logger.warning(f"Error parsing SEC-API transaction for {symbol}: {e}")
+                        continue
+
+            start_index += limit
+
+            # Stop if we got fewer results than requested (last page)
+            if len(data['transactions']) < limit:
+                break
+
+        logger.info(f"Retrieved {len(all_trades)} trades from SEC-API for {symbol}.")
+        return all_trades
+    '''
+
+    async def get_sec_api_insider_trades(self, session: aiohttp.ClientSession,
+                                         symbol: str, limit: int = 50,
+                                         filter_days: int = 30) -> List[InsiderTrade]:
+        """Get insider trades from SEC-API using a POST request with proper query structure."""
+        all_trades = []
+
+        if limit > 50:
+            limit = 50
+
+        to_date = datetime.now()
+        from_date = to_date - timedelta(days=filter_days)
+
+        start_index = 0
+
+        while True:
+            # Use simple query string format as shown in official Python SDK
+            query_payload = {
+                "query": f'issuer.tradingSymbol:"{symbol}" AND periodOfReport:[{from_date.strftime("%Y-%m-%d")} TO {to_date.strftime("%Y-%m-%d")}]',
+                "from": str(start_index),
+                "size": str(limit),
+                "sort": [{"filedAt": {"order": "desc"}}]
+            }
+
+            endpoint = "insider-trading"
+            logger.info(f"Requesting SEC-API data for {symbol}, starting at index {start_index}...")
+            data = await self._make_post_request(session, 'sec_api', endpoint, query_payload)
+
+            if not data or 'transactions' not in data or not data['transactions']:
+                break
+
+            for filing in data['transactions']:
+                # Check if nonDerivativeTable exists and has transactions
+                non_deriv_trans = filing.get('nonDerivativeTable', {}).get('transactions', [])
+                deriv_trans = filing.get('derivativeTable', {}).get('transactions', [])
+
+                transactions_list = non_deriv_trans + deriv_trans
+
+                for transaction in transactions_list:
+                    try:
+                        trans_code = transaction.get('coding', {}).get('code', '')
+                        if trans_code in ('A', 'P'):
+                            trans_type = TransactionType.BUY.value
+                        elif trans_code in ('D', 'S'):
+                            trans_type = TransactionType.SELL.value
+                        else:
+                            continue
+
+                        shares = transaction.get('amounts', {}).get('shares')
+                        price_per_share = transaction.get('amounts', {}).get('pricePerShare')
+
+                        if shares is None or price_per_share is None:
+                            continue
+
+                        shares = float(shares)
+                        price_per_share = float(price_per_share)
+
+                        insider_trade = InsiderTrade(
+                            symbol=filing.get('issuer', {}).get('tradingSymbol', symbol),
+                            company_name=filing.get('issuer', {}).get('name', ''),
+                            insider_name=filing.get('reportingOwner', {}).get('name', ''),
+                            position=filing.get('reportingOwner', {}).get('relationship', {}).get('officerTitle',
+                                                                                                  'Insider'),
+                            transaction_date=transaction.get('transactionDate', ''),
+                            transaction_type=trans_type,
+                            shares=int(shares),
+                            price=price_per_share,
+                            value=shares * price_per_share,
+                            form_type=filing.get('documentType', ''),
+                            source='SEC-API',
+                            filing_date=filing.get('filedAt', '')
+                        )
+                        all_trades.append(insider_trade)
+
+                    except (ValueError, TypeError, AttributeError) as e:
+                        logger.warning(f"Error parsing SEC-API transaction for {symbol}: {e}")
+                        continue
+
+            start_index += limit
+
+            if len(data['transactions']) < limit:
+                break
+
+        logger.info(f"Retrieved {len(all_trades)} trades from SEC-API for {symbol}.")
+        return all_trades
 
     async def get_eod_insider_trades(self, session: aiohttp.ClientSession,
                                      symbol: str) -> List[InsiderTrade]:
@@ -393,8 +633,8 @@ class InsiderTradingAggregator:
         async with aiohttp.ClientSession(timeout=timeout) as session:
             # Create tasks for all API calls
             tasks = [
-                self.get_fmp_insider_trades(session, symbol, limit_per_api),
-                #self.get_sec_api_insider_trades(session, symbol, limit_per_api),
+                #self.get_fmp_insider_trades(session, symbol, limit_per_api),
+                self.get_sec_api_insider_trades(session, symbol, limit_per_api),
                 #self.get_eod_insider_trades(session, symbol),
                 #self.get_tradefeeds_insider_trades(session, symbol, limit_per_api),
                 #self.get_factored_ai_insider_trades(session, symbol)

src/telegram_bot/config.py

@@ -17,6 +17,7 @@ class Config:
     OPENROUTER_API_KEY_2 = os.getenv('OPENROUTER_API_TOKEN_2', '')
     GEMINI_API_KEY = os.getenv('GEMINI_API_TOKEN', '')
     FMP_API_KEY = os.getenv('FMP_API_TOKEN', '')
+    SEC_API_KEY = os.getenv('SEC_API_TOKEN', '')
 
     @classmethod
     def validate(cls) -> bool: