Updated assessment and solutions

notebooks/assesment.ipynb

@@ -87,8 +87,8 @@
     "# spark_df = ?\n",
     "\n",
     "# Check schema and row count\n",
-    "# spark_df.printSchema()\n",
-    "# print(\"spark_df count:\", spark_df.count())\n"
+    "# spark_df. ...\n",
+    "# print(\"spark_df count:\", spark_df. ...)\n"
    ]
   },
   {
@@ -99,19 +99,19 @@
    "source": [
     "# 3. Split Data into Two Subsets for Merging/Joining\n",
     "# ==================================================\n",
-    "# Instead of using a second CSV, we'll simulate it by splitting the original dataset\n",
-    "# into two DataFrames:\n",
+    "# Split the dataset into two df's by column, then merge them \n",
+    "# back together\n",
     "#   df_part1: subset of columns -> PassengerId, Name, Sex, Age\n",
     "#   df_part2: subset of columns -> PassengerId, Fare, Survived, Pclass\n",
     "#\n",
-    "# We then merge these two separate DataFrames in both Pandas and Spark.\n",
+    "# \n",
     "\n",
     "# 3.1 Pandas Split\n",
     "# ----------------\n",
     "\n",
     "# TODO: Create two new DataFrames from pd_df:\n",
     "#    pd_part1 = pd_df[[\"PassengerId\", \"Name\", \"Sex\", \"Age\"]]\n",
-    "#    pd_part2 = pd_df[[\"PassengerId\", \"Fare\", \"Survived\", \"Pclass\"]]\n",
+    "#    pd_part2 = pd_df[...]\n",
     "\n",
     "# pd_part1 = ?\n",
     "# pd_part2 = ?\n",
@@ -129,8 +129,8 @@
     "# 3.2 Spark Split\n",
     "# ---------------\n",
     "# TODO: Create two new DataFrames from spark_df:\n",
-    "#    spark_part1 = spark_df.select(\"PassengerId\", \"Name\", \"Sex\", \"Age\")\n",
-    "#    spark_part2 = spark_df.select(\"PassengerId\", \"Fare\", \"Survived\", \"Pclass\")\n",
+    "#    spark_part1 = spark_df. ...\n",
+    "#    spark_part2 = spark_df. ...\n",
     "\n",
     "# spark_part1 = ?\n",
     "# spark_part2 = ?\n",
@@ -153,7 +153,6 @@
     "# TODO: Merge pd_part1 and pd_part2 on \"PassengerId\"\n",
     "# We'll call the merged DataFrame \"pd_merged\".\n",
     "#\n",
-    "# pd_merged = pd_part1.merge(pd_part2, on=\"PassengerId\", how=\"inner\")\n",
     "\n",
     "# pd_merged = ?\n",
     "# print(\"pd_merged shape:\", pd_merged.shape)\n",
@@ -171,8 +170,9 @@
     "# TODO: Join spark_part1 with spark_part2 on \"PassengerId\"\n",
     "# We'll call the joined DataFrame \"spark_merged\".\n",
     "#\n",
-    "# spark_merged = spark_part1.join(spark_part2, on=\"PassengerId\", how=\"inner\")\n",
     "\n",
+    "\n",
+    "#Uncomment below\n",
     "# spark_merged = ?\n",
     "# print(\"spark_merged count:\", spark_merged.count())\n",
     "# spark_merged.show(5)\n",
@@ -242,7 +242,7 @@
    "outputs": [],
    "source": [
     "# 6.2 TODO: Spark - Survival rate by Sex and Pclass\n",
-    "# e.g. groupBy(\"Sex\", \"Pclass\").agg(F.avg(\"Survived\"))\n",
+    "# Average survival rate by Sex and Pclass\n",
     "#\n",
     "# spark_survival_rate = ?\n",
     "# spark_survival_rate.show()\n"
@@ -261,11 +261,20 @@
     "# 7.1 TODO: Write spark_merged_clean to Parquet\n",
     "# e.g., spark_merged_clean.write. ...\n",
     "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
     "# 7.2 TODO: Read it back into a new Spark DataFrame called 'spark_parquet_df'\n",
     "# spark_parquet_df = ?\n",
     "\n",
     "# print(\"spark_parquet_df count:\", spark_parquet_df.count())\n",
-    "# spark_parquet_df.show(5)\n"
+    "# spark_parquet_df.show(5)"
    ]
   },
   {
@@ -274,14 +283,21 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# 8. Bonus 1: Create a Temp View and Query\n",
+    "# 8. Create a Temp View and Query\n",
     "# ========================================\n",
     "# 8.1 TODO: Create a temp view with 'spark_merged_clean' (e.g. \"titanic_merged\")\n",
     "# spark_merged_clean.createOrReplaceTempView(\"titanic_merged\")\n",
     "\n",
-    "# 8.2 TODO: Spark SQL query example\n",
+    "# 8.2 TODO: Spark SQL query examples\n",
+    "\n",
+    "#Get the average passenger age grouped by PClass\n",
     "# result_df = spark.sql(\"SELECT ... FROM titanic_merged GROUP BY ...\")\n",
-    "# result_df.show()\n"
+    "# result_df.show()\n",
+    "\n",
+    "# Calculate the Pearson correlation between passenger Fare and Survival\n",
+    "# using either SQL or another method\n",
+    "# Corr.(X, Y) = cov(X,Y)/(std(X)*std(Y))\n",
+    "# corr = ..."
    ]
   },
   {

notebooks/solutions.ipynb

@@ -256,7 +256,22 @@
     "    GROUP BY Pclass\n",
     "    ORDER BY Pclass\n",
     "    \"\"\")\n",
-    "result_df.show()\n"
+    "result_df.show()\n",
+    "\n",
+    "#Correlation between Fare and Survival\n",
+    "# Compute the Pearson correlation between Fare and Survived\n",
+    "\n",
+    "correlation1 = spark_merged_clean.stat.corr(\"Fare\", \"Survived\", \"pearson\")\n",
+    "\n",
+    "print(\"Pearson correlation between Fare and Survived:\", correlation1)\n",
+    "\n",
+    "correlation2 = spark.sql(\n",
+    "    '''\n",
+    "    SELECT\n",
+    "    covar_samp(Fare, Survived) / (stddev_samp(Fare)*stddev_samp(Survived)) as correlation\n",
+    "    FROM titanic_merged_clean\n",
+    "    '''\n",
+    ")\n"
    ]
   },
   {