Add randomization to urls required to fetch all nearby pages

- If #required nearby pages < #available nearby pages, then randomize the urls in order to avoid biasing of results in one direction. Since the small circles are stacked in a hexagonal manner (and it stacks from south to north), not randomizing urls will result in cluttering of results in south with no result on north, in case # required pages < # available pages

- moved `fetch_url` to utils.py

- removed filtering based on distance on frontend side for nearby point rendering

backend/utils.py

@@ -1,4 +1,6 @@
 import math
+import httpx
+
 def generate_circle_centers(center_lat, center_lon, radius_km, small_radius_km=10):
     """
         Generate a list of centers of small circles (radius=10km) needed to cover a larger circle.
@@ -40,4 +42,28 @@ def generate_circle_centers(center_lat, center_lon, radius_km, small_radius_km=1
                 lon = center_lon + delta_lon
                 results.append((lat, lon))
     
-    return results
+    return results
+
+
+
+async def fetch_url(client: httpx.AsyncClient, url: str):
+    """
+        Fetch a URL asynchronously using httpx and return the response status and data.
+        This function is asynchrounously used to fetch multiple URLs in parallel when search radius > 10km.
+        Input:
+        - client: httpx.AsyncClient instance
+        - url: URL to fetch
+        Output:
+        - A dictionary with the URL, status code, and data if available.
+            - Data includes the JSON format of wiki geosearch response.
+        If an error occurs, return a dictionary with the URL and the error message.
+    """
+    try:
+        response = await client.get(url, timeout=10.0)
+        return {
+            "url": url,
+            "status": response.status_code,
+            "data": response.json() if response.status_code == 200 else None,
+        }
+    except Exception as e:
+        return {"url": url, "error": str(e)}

frontend/src/components/Map.js

@@ -247,9 +247,7 @@ const Map = ( { onMapClick, searchQuery, contentType, setSearchQuery, setSubmitt
                 
                 if (res.ok) {
                     const data = await res.json();
-                    const markers = data.pages.filter(
-                                page => typeof page.dist === "number" && page.dist <= explorationRadius * 1000
-                            ).map(page => ({
+                    const markers = data.pages.map(page => ({
                         position: [page.lat, page.lon],
                         title: page.title,
                         distance: page.dist

main.py

@@ -8,7 +8,8 @@ import geopy.distance
 from cachetools import TTLCache
 import os
 from dotenv import load_dotenv
-from backend.utils import generate_circle_centers
+from random import shuffle
+from backend.utils import generate_circle_centers, fetch_url
 
 load_dotenv()
 
@@ -175,16 +176,7 @@ def get_geodistance(payload: Geodistance):
     )
 
 
-async def fetch_url(client: httpx.AsyncClient, url: str):
-    try:
-        response = await client.get(url, timeout=10.0)
-        return {
-            "url": url,
-            "status": response.status_code,
-            "data": response.json() if response.status_code == 200 else None,
-        }
-    except Exception as e:
-        return {"url": url, "error": str(e)}
+
     
 @app.post("/wiki/nearby")
 async def get_nearby_wiki_pages(payload: NearbyWikiPage):
@@ -217,7 +209,9 @@ async def get_nearby_wiki_pages(payload: NearbyWikiPage):
     radius = payload.radius
     limit = payload.limit
 
-    if radius <= 10000:
+    wiki_geosearch_radius_limit_meters = 10000 # Wikipedia API limit for geosearch radius in meters
+
+    if radius <= wiki_geosearch_radius_limit_meters:
         url = ("https://en.wikipedia.org/w/api.php"+"?action=query"
                 "&list=geosearch"
                 f"&gscoord={lat_center}|{lon_center}"
@@ -248,11 +242,14 @@ async def get_nearby_wiki_pages(payload: NearbyWikiPage):
                 content={"error": str(e)},
                 status_code=500
             )
-    elif radius > 10000:
+        
+    elif radius > wiki_geosearch_radius_limit_meters:
+        print(radius)
         small_circle_centers = generate_circle_centers(lat_center, lon_center, radius / 1000, small_radius_km=10)
         all_pages = []
         base_url = "https://en.wikipedia.org/w/api.php?action=query&list=geosearch&gscoord={lat}|{lon}&gsradius={small_radius_km}&gslimit={page_limit}&format=json"
-        urls = [base_url.format(lat=center[0], lon=center[1], small_radius_km=10*1000, page_limit=100) for center in small_circle_centers]
+        urls = [base_url.format(lat=center[0], lon=center[1], small_radius_km=wiki_geosearch_radius_limit_meters, page_limit=100) for center in small_circle_centers]
+        shuffle(urls) # If # available pages > # requested pages by user, randomize the results to avoid clustering around a single direction.
 
         print("URL Counts:", len(urls))
         try:
@@ -263,18 +260,25 @@ async def get_nearby_wiki_pages(payload: NearbyWikiPage):
             # print(results)
             for result in results:
                 for unit in result.get("data", {}).get("query", {}).get("geosearch", []):
+                    if len(all_pages) >= limit:
+                        break
                     lat, lon = unit.get("lat"), unit.get("lon")
                     if lat is not None and lon is not None:
                         dist = int(geopy.distance.distance(
                                 (lat_center, lon_center), (lat, lon)
                             ).m)
-                        print(dist)
+                        # print(dist)
                     else: 
                         dist = None
 
+                    if (not dist) or (dist and dist > radius):
+                        continue
+
                     unit_with_dist = {**unit, "dist": dist}
                     all_pages.append(unit_with_dist)
 
+            # print(all_pages)
+
             return JSONResponse(
                 content={
                     "pages": all_pages,