SPARK DataFrame.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "Py4JJavaError",
     "evalue": "An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.\n: java.lang.IllegalAccessError: class org.apache.spark.storage.StorageUtils$ (in unnamed module @0x37858383) cannot access class sun.nio.ch.DirectBuffer (in module java.base) because module java.base does not export sun.nio.ch to unnamed module @0x37858383\r\n\tat org.apache.spark.storage.StorageUtils$.<init>(StorageUtils.scala:213)\r\n\tat org.apache.spark.storage.StorageUtils$.<clinit>(StorageUtils.scala)\r\n\tat org.apache.spark.storage.BlockManagerMasterEndpoint.<init>(BlockManagerMasterEndpoint.scala:110)\r\n\tat org.apache.spark.SparkEnv$.$anonfun$create$9(SparkEnv.scala:348)\r\n\tat org.apache.spark.SparkEnv$.registerOrLookupEndpoint$1(SparkEnv.scala:287)\r\n\tat org.apache.spark.SparkEnv$.create(SparkEnv.scala:336)\r\n\tat org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:191)\r\n\tat org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277)\r\n\tat org.apache.spark.SparkContext.<init>(SparkContext.scala:460)\r\n\tat org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)\r\n\tat java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)\r\n\tat java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)\r\n\tat java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:238)\r\n\tat py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)\r\n\tat py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)\r\n\tat py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\r\n\tat py4j.ClientServerConnection.run(ClientServerConnection.java:106)\r\n\tat java.base/java.lang.Thread.run(Thread.java:833)\r\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\SPARK DataFrame.ipynb Cell 2'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/duimh/Documents/GitHub/Python-spielerei/SPARK%20DataFrame.ipynb#ch0000001?line=0'>1</a>\u001b[0m spark \u001b[39m=\u001b[39m SparkSession\u001b[39m.\u001b[39;49mbuilder\u001b[39m.\u001b[39;49mmaster(\u001b[39m\"\u001b[39;49m\u001b[39mlocal[*]\u001b[39;49m\u001b[39m\"\u001b[39;49m)\u001b[39m.\u001b[39;49mappName(\u001b[39m\"\u001b[39;49m\u001b[39mGreat Expectations with Pandas DataFrame\u001b[39;49m\u001b[39m\"\u001b[39;49m)\u001b[39m.\u001b[39;49mgetOrCreate()\n",
      "File \u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\.venv\\lib\\site-packages\\pyspark\\sql\\session.py:228\u001b[0m, in \u001b[0;36mSparkSession.Builder.getOrCreate\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/sql/session.py?line=225'>226</a>\u001b[0m         sparkConf\u001b[39m.\u001b[39mset(key, value)\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/sql/session.py?line=226'>227</a>\u001b[0m     \u001b[39m# This SparkContext may be an existing one.\u001b[39;00m\n\u001b[1;32m--> <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/sql/session.py?line=227'>228</a>\u001b[0m     sc \u001b[39m=\u001b[39m SparkContext\u001b[39m.\u001b[39;49mgetOrCreate(sparkConf)\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/sql/session.py?line=228'>229</a>\u001b[0m \u001b[39m# Do not update `SparkConf` for existing `SparkContext`, as it's shared\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/sql/session.py?line=229'>230</a>\u001b[0m \u001b[39m# by all sessions.\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/sql/session.py?line=230'>231</a>\u001b[0m session \u001b[39m=\u001b[39m SparkSession(sc)\n",
      "File \u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\.venv\\lib\\site-packages\\pyspark\\context.py:392\u001b[0m, in \u001b[0;36mSparkContext.getOrCreate\u001b[1;34m(cls, conf)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=389'>390</a>\u001b[0m \u001b[39mwith\u001b[39;00m SparkContext\u001b[39m.\u001b[39m_lock:\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=390'>391</a>\u001b[0m     \u001b[39mif\u001b[39;00m SparkContext\u001b[39m.\u001b[39m_active_spark_context \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m--> <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=391'>392</a>\u001b[0m         SparkContext(conf\u001b[39m=\u001b[39;49mconf \u001b[39mor\u001b[39;49;00m SparkConf())\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=392'>393</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m SparkContext\u001b[39m.\u001b[39m_active_spark_context\n",
      "File \u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\.venv\\lib\\site-packages\\pyspark\\context.py:146\u001b[0m, in \u001b[0;36mSparkContext.__init__\u001b[1;34m(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=143'>144</a>\u001b[0m SparkContext\u001b[39m.\u001b[39m_ensure_initialized(\u001b[39mself\u001b[39m, gateway\u001b[39m=\u001b[39mgateway, conf\u001b[39m=\u001b[39mconf)\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=144'>145</a>\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[1;32m--> <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=145'>146</a>\u001b[0m     \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=146'>147</a>\u001b[0m                   conf, jsc, profiler_cls)\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=147'>148</a>\u001b[0m \u001b[39mexcept\u001b[39;00m:\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=148'>149</a>\u001b[0m     \u001b[39m# If an error occurs, clean up in order to allow future SparkContext creation:\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=149'>150</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mstop()\n",
      "File \u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\.venv\\lib\\site-packages\\pyspark\\context.py:209\u001b[0m, in \u001b[0;36mSparkContext._do_init\u001b[1;34m(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=205'>206</a>\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39menvironment[\u001b[39m\"\u001b[39m\u001b[39mPYTHONHASHSEED\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m os\u001b[39m.\u001b[39menviron\u001b[39m.\u001b[39mget(\u001b[39m\"\u001b[39m\u001b[39mPYTHONHASHSEED\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39m0\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=207'>208</a>\u001b[0m \u001b[39m# Create the Java SparkContext through Py4J\u001b[39;00m\n\u001b[1;32m--> <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=208'>209</a>\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_jsc \u001b[39m=\u001b[39m jsc \u001b[39mor\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_initialize_context(\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_conf\u001b[39m.\u001b[39;49m_jconf)\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=209'>210</a>\u001b[0m \u001b[39m# Reset the SparkConf to the one actually used by the SparkContext in JVM.\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=210'>211</a>\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_conf \u001b[39m=\u001b[39m SparkConf(_jconf\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_jsc\u001b[39m.\u001b[39msc()\u001b[39m.\u001b[39mconf())\n",
      "File \u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\.venv\\lib\\site-packages\\pyspark\\context.py:329\u001b[0m, in \u001b[0;36mSparkContext._initialize_context\u001b[1;34m(self, jconf)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=324'>325</a>\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_initialize_context\u001b[39m(\u001b[39mself\u001b[39m, jconf):\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=325'>326</a>\u001b[0m     \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=326'>327</a>\u001b[0m \u001b[39m    Initialize SparkContext in function to allow subclass specific initialization\u001b[39;00m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=327'>328</a>\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/pyspark/context.py?line=328'>329</a>\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_jvm\u001b[39m.\u001b[39;49mJavaSparkContext(jconf)\n",
      "File \u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\.venv\\lib\\site-packages\\py4j\\java_gateway.py:1585\u001b[0m, in \u001b[0;36mJavaClass.__call__\u001b[1;34m(self, *args)\u001b[0m\n\u001b[0;32m   <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1578'>1579</a>\u001b[0m command \u001b[39m=\u001b[39m proto\u001b[39m.\u001b[39mCONSTRUCTOR_COMMAND_NAME \u001b[39m+\u001b[39m\\\n\u001b[0;32m   <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1579'>1580</a>\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_command_header \u001b[39m+\u001b[39m\\\n\u001b[0;32m   <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1580'>1581</a>\u001b[0m     args_command \u001b[39m+\u001b[39m\\\n\u001b[0;32m   <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1581'>1582</a>\u001b[0m     proto\u001b[39m.\u001b[39mEND_COMMAND_PART\n\u001b[0;32m   <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1583'>1584</a>\u001b[0m answer \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_gateway_client\u001b[39m.\u001b[39msend_command(command)\n\u001b[1;32m-> <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1584'>1585</a>\u001b[0m return_value \u001b[39m=\u001b[39m get_return_value(\n\u001b[0;32m   <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1585'>1586</a>\u001b[0m     answer, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_gateway_client, \u001b[39mNone\u001b[39;49;00m, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_fqn)\n\u001b[0;32m   <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1587'>1588</a>\u001b[0m \u001b[39mfor\u001b[39;00m temp_arg \u001b[39min\u001b[39;00m temp_args:\n\u001b[0;32m   <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/java_gateway.py?line=1588'>1589</a>\u001b[0m     temp_arg\u001b[39m.\u001b[39m_detach()\n",
      "File \u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\.venv\\lib\\site-packages\\py4j\\protocol.py:326\u001b[0m, in \u001b[0;36mget_return_value\u001b[1;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=323'>324</a>\u001b[0m value \u001b[39m=\u001b[39m OUTPUT_CONVERTER[\u001b[39mtype\u001b[39m](answer[\u001b[39m2\u001b[39m:], gateway_client)\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=324'>325</a>\u001b[0m \u001b[39mif\u001b[39;00m answer[\u001b[39m1\u001b[39m] \u001b[39m==\u001b[39m REFERENCE_TYPE:\n\u001b[1;32m--> <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=325'>326</a>\u001b[0m     \u001b[39mraise\u001b[39;00m Py4JJavaError(\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=326'>327</a>\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mAn error occurred while calling \u001b[39m\u001b[39m{0}\u001b[39;00m\u001b[39m{1}\u001b[39;00m\u001b[39m{2}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=327'>328</a>\u001b[0m         \u001b[39mformat\u001b[39m(target_id, \u001b[39m\"\u001b[39m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m, name), value)\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=328'>329</a>\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=329'>330</a>\u001b[0m     \u001b[39mraise\u001b[39;00m Py4JError(\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=330'>331</a>\u001b[0m         \u001b[39m\"\u001b[39m\u001b[39mAn error occurred while calling \u001b[39m\u001b[39m{0}\u001b[39;00m\u001b[39m{1}\u001b[39;00m\u001b[39m{2}\u001b[39;00m\u001b[39m. Trace:\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39m{3}\u001b[39;00m\u001b[39m\\n\u001b[39;00m\u001b[39m\"\u001b[39m\u001b[39m.\u001b[39m\n\u001b[0;32m    <a href='file:///c%3A/Users/duimh/Documents/GitHub/Python-spielerei/.venv/lib/site-packages/py4j/protocol.py?line=331'>332</a>\u001b[0m         \u001b[39mformat\u001b[39m(target_id, \u001b[39m\"\u001b[39m\u001b[39m.\u001b[39m\u001b[39m\"\u001b[39m, name, value))\n",
      "\u001b[1;31mPy4JJavaError\u001b[0m: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.\n: java.lang.IllegalAccessError: class org.apache.spark.storage.StorageUtils$ (in unnamed module @0x37858383) cannot access class sun.nio.ch.DirectBuffer (in module java.base) because module java.base does not export sun.nio.ch to unnamed module @0x37858383\r\n\tat org.apache.spark.storage.StorageUtils$.<init>(StorageUtils.scala:213)\r\n\tat org.apache.spark.storage.StorageUtils$.<clinit>(StorageUtils.scala)\r\n\tat org.apache.spark.storage.BlockManagerMasterEndpoint.<init>(BlockManagerMasterEndpoint.scala:110)\r\n\tat org.apache.spark.SparkEnv$.$anonfun$create$9(SparkEnv.scala:348)\r\n\tat org.apache.spark.SparkEnv$.registerOrLookupEndpoint$1(SparkEnv.scala:287)\r\n\tat org.apache.spark.SparkEnv$.create(SparkEnv.scala:336)\r\n\tat org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:191)\r\n\tat org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:277)\r\n\tat org.apache.spark.SparkContext.<init>(SparkContext.scala:460)\r\n\tat org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)\r\n\tat java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)\r\n\tat java.base/jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77)\r\n\tat java.base/jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)\r\n\tat java.base/java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499)\r\n\tat java.base/java.lang.reflect.Constructor.newInstance(Constructor.java:480)\r\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)\r\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\r\n\tat py4j.Gateway.invoke(Gateway.java:238)\r\n\tat py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)\r\n\tat py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)\r\n\tat py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)\r\n\tat py4j.ClientServerConnection.run(ClientServerConnection.java:106)\r\n\tat java.base/java.lang.Thread.run(Thread.java:833)\r\n"
     ]
    }
   ],
   "source": [
    "spark = SparkSession.builder.master(\"local[*]\").appName(\"Great Expectations with Pandas DataFrame\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'spark' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\SPARK DataFrame.ipynb Cell 3'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/duimh/Documents/GitHub/Python-spielerei/SPARK%20DataFrame.ipynb#ch0000002?line=0'>1</a>\u001b[0m raw_df \u001b[39m=\u001b[39m spark\u001b[39m.\u001b[39mread\u001b[39m.\u001b[39moption(\u001b[39m\"\u001b[39m\u001b[39mheader\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mTrue\u001b[39;00m)\u001b[39m.\u001b[39mcsv(\u001b[39m\"\u001b[39m\u001b[39m../Kickstarter_projects_Feb19.csv\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m      <a href='vscode-notebook-cell:/c%3A/Users/duimh/Documents/GitHub/Python-spielerei/SPARK%20DataFrame.ipynb#ch0000002?line=1'>2</a>\u001b[0m raw_df\u001b[39m.\u001b[39mcreateOrReplaceTempView(\u001b[39m\"\u001b[39m\u001b[39mCAMPAIGNS\u001b[39m\u001b[39m\"\u001b[39m)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'spark' is not defined"
     ]
    }
   ],
   "source": [
    "raw_df = spark.read.option(\"header\", True).csv(\"../Kickstarter_projects_Feb19.csv\")\n",
    "raw_df.createOrReplaceTempView(\"CAMPAIGNS\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'raw_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\SPARK DataFrame.ipynb Cell 4'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/duimh/Documents/GitHub/Python-spielerei/SPARK%20DataFrame.ipynb#ch0000003?line=0'>1</a>\u001b[0m \u001b[39mtype\u001b[39m(raw_df)\n",
      "\u001b[1;31mNameError\u001b[0m: name 'raw_df' is not defined"
     ]
    }
   ],
   "source": [
    "type(raw_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'raw_df' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32mc:\\Users\\duimh\\Documents\\GitHub\\Python-spielerei\\SPARK DataFrame.ipynb Cell 5'\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> <a href='vscode-notebook-cell:/c%3A/Users/duimh/Documents/GitHub/Python-spielerei/SPARK%20DataFrame.ipynb#ch0000004?line=0'>1</a>\u001b[0m raw_df\u001b[39m.\u001b[39mprintSchema()\n",
      "\u001b[1;31mNameError\u001b[0m: name 'raw_df' is not defined"
     ]
    }
   ],
   "source": [
    "raw_df.printSchema()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "2d9d1199941cbf654485e01298434612bb0eff62e210196293117e958bbe9a1e"
  },
  "kernelspec": {
   "display_name": "Python 3.10.4 ('.venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}