{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3928f35-85c4-4763-9e51-cc3af482b5d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "# Replace these with your AWS credentials and S3 bucket name\n",
    "aws_access_key_id = \"YOUR_AWS_ACCESS_KEY\"\n",
    "aws_secret_access_key = \"YOUR_AWS_SECRET_KEY\"\n",
    "bucket = \"your-bucket-name\"\n",
    "\n",
    "# Create Spark session with S3 support\n",
    "spark = SparkSession.builder \\\n",
    ".appName(\"S3 Join Example\") \\\n",
    ".config(\"spark.hadoop.fs.s3a.access.key\", aws_access_key_id) \\\n",
    ".config(\"spark.hadoop.fs.s3a.secret.key\", aws_secret_access_key) \\\n",
    ".config(\"spark.hadoop.fs.s3a.endpoint\", \"s3.amazonaws.com\") \\\n",
    ".config(\"spark.hadoop.fs.s3a.impl\", \"org.apache.hadoop.fs.s3a.S3AFileSystem\") \\\n",
    ".getOrCreate()\n",
    "\n",
    "# Read CSVs from S3\n",
    "browser_csv = spark.read.csv(f\"s3a://{bucket}/syn-demo/Browser.csv\", header=True)\n",
    "country_csv = spark.read.csv(f\"s3a://{bucket}/syn-demo/country.csv\", header=True)\n",
    "\n",
    "# Filter DataFrames\n",
    "browser_csv = browser_csv.filter(browser_csv['browser'] == 'safari')\n",
    "country_csv = country_csv.filter(country_csv['country'] == 'usa')\n",
    "\n",
    "# Show filtered data\n",
    "print(\"Filtered Browser Data:\")\n",
    "browser_csv.show(10, False)\n",
    "\n",
    "print(\"Filtered Country Data:\")\n",
    "country_csv.show(10, False)\n",
    "\n",
    "# Join the data on visitor_id\n",
    "joined_df = browser_csv.join(country_csv, on='visitor_id', how='left')\n",
    "\n",
    "# Show sample of the joined data\n",
    "print(\"Joined Data:\")\n",
    "joined_df.show(10, False)\n",
    "\n",
    "# Save the joined output back to S3 (coalesce to 1 file if needed)\n",
    "output_path = f\"s3a://{bucket}/syn-demo/test_joined_output\"\n",
    "joined_df.coalesce(1).write.mode(\"overwrite\").option(\"header\", \"true\").csv(output_path)\n",
    "\n",
    "print(\"✅ Done! Joined data written to S3.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10",
   "language": "python",
   "name": "syntasa_kernel_python_310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}