Fix sensitive data columns

murilobellatini · Sep 7, 2020 · 7183a79 · 7183a79
1 parent 8ad2a94
commit 7183a79
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 6 deletions.
diff --git a/dev/docker-volume/notebooks/04-create-trusted-data-order-Copy1.ipynb b/dev/docker-volume/notebooks/04-create-trusted-data-order-Copy1.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 27,
+   "execution_count": 31,
    "metadata": {},
    "outputs": [
     {
@@ -24,7 +24,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 28,
+   "execution_count": 32,
    "metadata": {},
    "outputs": [
     {
@@ -42,7 +42,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 29,
+   "execution_count": 33,
    "metadata": {},
    "outputs": [
     {
@@ -61,7 +61,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 26,
+   "execution_count": 34,
    "metadata": {},
    "outputs": [
     {
@@ -70,7 +70,7 @@
        "pyspark.sql.session.SparkSession"
       ]
      },
-     "execution_count": 26,
+     "execution_count": 34,
      "metadata": {},
      "output_type": "execute_result"
     }

diff --git a/dev/docker-volume/src/DataProcessor.py b/dev/docker-volume/src/DataProcessor.py
@@ -123,7 +123,7 @@ def create_trusted_order(spark:SparkSession):
     output_path = TRUSTED_DATA_PATH / 'order'
 
     # anonymize sensitive data by dropping columns
-    sensitive_data_columns = ['cpf', 'customer_name', 'consumer_customer_name', 'consumer_customer_phone_number']
+    sensitive_data_columns = ['order_cpf', 'order_customer_name', 'consumer_customer_name', 'consumer_customer_phone_number']
     tmp = tmp.drop(*sensitive_data_columns)
 
     tmp.write.parquet(str(output_path))

diff --git a/dev/docker-volume/src/__pycache__/DataProcessor.cpython-38.pyc b/dev/docker-volume/src/__pycache__/DataProcessor.cpython-38.pyc