Upload folder using huggingface_hub

__init__.py

@@ -0,0 +1,9 @@
+# AnonySpark: Lightweight PySpark data anonymization
+from .masking import (
+    mask_email, mask_name, mask_date,
+    mask_ssn, mask_itin, mask_phone,
+    mask_email_udf, mask_name_udf, mask_date_udf,
+    mask_ssn_udf, mask_itin_udf, mask_phone_udf
+)
+
+from .utils import apply_masking

cli.py

@@ -0,0 +1,57 @@
+
+import argparse
+import json
+import os
+from pyspark.sql import SparkSession
+from anonyspark.masking import (
+    mask_email_udf, mask_name_udf, mask_date_udf,
+    mask_ssn_udf, mask_itin_udf, mask_phone_udf
+)
+
+def apply_masking(df, schema):
+    """
+    Apply masking UDFs based on schema definitions.
+    """
+    for column, dtype in schema.items():
+        if dtype == "email":
+            df = df.withColumn(f"masked_{column}", mask_email_udf(df[column]))
+        elif dtype == "name":
+            df = df.withColumn(f"masked_{column}", mask_name_udf(df[column]))
+        elif dtype == "dob":
+            df = df.withColumn(f"masked_{column}", mask_date_udf(df[column]))
+        elif dtype == "ssn":
+            df = df.withColumn(f"masked_{column}", mask_ssn_udf(df[column]))
+        elif dtype == "itin":
+            df = df.withColumn(f"masked_{column}", mask_itin_udf(df[column]))
+        elif dtype == "phone":
+            df = df.withColumn(f"masked_{column}", mask_phone_udf(df[column]))
+    return df
+
+def main():
+    parser = argparse.ArgumentParser(description="AnonySpark CLI for masking sensitive data.")
+    parser.add_argument('--input', type=str, required=True, help='Path to input CSV file')
+    parser.add_argument('--output', type=str, required=True, help='Directory to save masked output')
+    parser.add_argument('--schema', type=str, required=True, help='Path to masking schema JSON file')
+    args = parser.parse_args()
+
+    # Create output directory if it doesn't exist
+    os.makedirs(args.output, exist_ok=True)
+
+    # Start Spark
+    spark = SparkSession.builder.master("local[*]").appName("AnonysparkCLI").getOrCreate()
+
+    # Load data and schema
+    df = spark.read.csv(args.input, header=True)
+    with open(args.schema, 'r') as f:
+        schema = json.load(f)
+
+    # Apply masking
+    masked_df = apply_masking(df, schema)
+
+    # Save to output directory
+    masked_df.write.mode("overwrite").csv(args.output, header=True)
+
+    print(f"Masked file written to: {args.output}")
+
+if __name__ == "__main__":
+    main()

masking.py

@@ -0,0 +1,52 @@
+__all__ = [
+    "mask_email_udf", "mask_name_udf", "mask_date_udf",
+    "mask_ssn_udf", "mask_itin_udf", "mask_phone_udf"
+]
+
+from pyspark.sql.functions import udf
+from pyspark.sql.types import StringType
+import re
+from datetime import datetime
+
+# Masking functions
+def mask_email(value):
+    if value and "@" in value:
+        user, domain = value.split("@")
+        return "***@" + domain
+    return None
+
+def mask_name(value):
+    if value:
+        return value[0] + "***"
+    return None
+
+def mask_date(value):
+    try:
+        dt = datetime.strptime(value, "%Y-%m-%d")
+        return dt.strftime("***-**-%d")
+    except:
+        return None
+
+def mask_ssn(value):
+    if value and re.match(r"\d{3}-\d{2}-\d{4}", value):
+        return "***-**-" + value[-4:]
+    return None
+
+def mask_itin(value):
+    if value and re.match(r"9\d{2}-7\d-\d{4}", value):
+        return "***-**-" + value[-4:]
+    return None
+
+def mask_phone(value):
+    if value and re.match(r"\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}", value):
+        return "***-***-" + value[-4:]
+    return None
+
+# UDFs for Spark
+mask_email_udf = udf(mask_email, StringType())
+mask_name_udf = udf(mask_name, StringType())
+mask_date_udf = udf(mask_date, StringType())
+mask_ssn_udf = udf(mask_ssn, StringType())
+mask_itin_udf = udf(mask_itin, StringType())
+mask_phone_udf = udf(mask_phone, StringType())
+

utils.py

@@ -0,0 +1,25 @@
+from pyspark.sql.functions import col
+
+def apply_masking(df, schema):
+    """
+    Apply masking UDFs to specified columns based on schema.
+    Schema = { "original_col": "mask_type" }
+    """
+    from .masking import (
+        mask_email_udf, mask_name_udf, mask_date_udf,
+        mask_ssn_udf, mask_itin_udf, mask_phone_udf
+    )
+
+    masking_map = {
+        "email": mask_email_udf,
+        "name": mask_name_udf,
+        "dob": mask_date_udf,
+        "ssn": mask_ssn_udf,
+        "itin": mask_itin_udf,
+        "phone": mask_phone_udf,
+    }
+
+    for col_name, mask_type in schema.items():
+        if mask_type in masking_map:
+            df = df.withColumn(f"masked_{col_name}", masking_map[mask_type](col(col_name)))
+    return df