From fb63c6ad952e218d298c96de8f2b236fd801e1e1 Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Thu, 25 Jan 2024 23:17:10 +0100 Subject: [PATCH 1/9] changes to preview table --- backend/routes/data_profile_routes.py | 1 + .../pages/upload/CreateDataProfileWindow.jsx | 17 ++- .../upload/DataPreviewAndSchemaEditor.jsx | 133 ++++++++++++++++++ 3 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx diff --git a/backend/routes/data_profile_routes.py b/backend/routes/data_profile_routes.py index 4d6cb72..1a877c4 100644 --- a/backend/routes/data_profile_routes.py +++ b/backend/routes/data_profile_routes.py @@ -43,6 +43,7 @@ async def get_data_profiles_by_org_id(current_user: User = Depends(get_current_u async def save_data_profile( request: DataProfileCreateRequest, current_user: User = Depends(get_current_user) ) -> DataProfileCreateResponse: + """Save a new data profile to the database""" with DatabaseManager() as session: data_profile_manager = DataProfileManager(session) if data_profile_manager.get_dataprofile_by_name_and_org( diff --git a/frontend/src/pages/upload/CreateDataProfileWindow.jsx b/frontend/src/pages/upload/CreateDataProfileWindow.jsx index 12748e8..3ed25ee 100644 --- a/frontend/src/pages/upload/CreateDataProfileWindow.jsx +++ b/frontend/src/pages/upload/CreateDataProfileWindow.jsx @@ -11,7 +11,7 @@ import { } from "@mui/material"; import axios from "axios"; import FileUploader from "./FileUploader"; -import PreviewTable from "./PreviewTable"; +import DataPreviewAndSchemaEditor from "./DataPreviewAndSchemaEditor"; import { API_URL } from "../../utils/constants"; function CreateDataProfileWindow({ open, onClose, onCreate }) { @@ -29,7 +29,6 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { const handlePreview = () => { if (sampleFiles.length && extractInstructions) { - console.log(extractInstructions); setIsPreviewLoading(true); const formData = new FormData(); sampleFiles.forEach((file) => { @@ -85,7 +84,9 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { /> - {previewData && } + {previewData && ( + + )} {isPreviewLoading && } @@ -98,6 +99,11 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { disabled={ !sampleFiles || !extractInstructions || isPreviewLoading } + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + } + }} > Preview @@ -106,6 +112,11 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { color="primary" variant="contained" disabled={!isPreviewTableOpen || !name || !extractInstructions} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + } + }} > Create diff --git a/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx b/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx new file mode 100644 index 0000000..4146b15 --- /dev/null +++ b/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx @@ -0,0 +1,133 @@ +import React, { useState, useEffect, useRef } from "react"; +import { + Box, + IconButton, + InputAdornment, + MenuItem, + Paper, + Select, + Table, + TableBody, + TableCell, + TableContainer, + TableHead, + TableRow, + TextField, +} from "@mui/material"; +import EditIcon from "@mui/icons-material/Edit"; + +function DataPreviewAndSchemaEditor({ previewData }) { + const data = Array.isArray(previewData) ? previewData : [previewData]; + const [columnNames, setColumnNames] = useState([]); + const [columnTypes, setColumnTypes] = useState([]); + const [editingColumnIndex, setEditingColumnIndex] = useState(null); + const inputRefs = useRef([]); + + useEffect(() => { + if (data && data.length > 0) { + const newColumnNames = Object.keys(data[0]); + const newColumnTypes = newColumnNames.map(() => "string"); + if (JSON.stringify(newColumnNames) !== JSON.stringify(columnNames)) { + setColumnNames(newColumnNames); + } + if (JSON.stringify(newColumnTypes) !== JSON.stringify(columnTypes)) { + setColumnTypes(newColumnTypes); + } + } + }, [data]); + + const generateHeaderRow = (data) => { + if (data && data.length > 0) { + return columnNames.map((key, index) => ( + + + handleColumnNameChange(index, event.target.value) + } + variant="standard" + InputProps={{ + disableUnderline: true, + readOnly: editingColumnIndex !== index, + endAdornment: ( + + handleEditClick(index)}> + + + + ), + }} + inputRef={(ref) => (inputRefs.current[index] = ref)} + onClick={() => handleEditClick(index)} + style={{ cursor: "pointer" }} + inputProps={{ + style: { + cursor: editingColumnIndex === index ? "text" : "pointer", + }, + }} + onKeyDown={(event) => { + if (event.key === "Enter") { + event.preventDefault(); + setEditingColumnIndex(null); + } + }} + /> + + + )); + } + }; + + const handleColumnTypeChange = (index, newType) => { + setColumnTypes((prevColumnTypes) => { + const newColumnTypes = [...prevColumnTypes]; + newColumnTypes[index] = newType; + return newColumnTypes; + }); + }; + + const handleEditClick = (index) => { + setEditingColumnIndex(index); + inputRefs.current[index].select(); + }; + + const handleColumnNameChange = (index, newName) => { + setColumnNames((prevColumnNames) => { + const newColumnNames = [...prevColumnNames]; + newColumnNames[index] = newName; + return newColumnNames; + }); + }; + + return ( + + + + {generateHeaderRow(data)} + + + {data.map((row, index) => ( + + {Object.values(row).map((value, idx) => ( + {value} + ))} + + ))} + +
+ + ); +} + +export default DataPreviewAndSchemaEditor; From 7fcaedd87d3990221a56fcd995f2e59b21a743c2 Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Fri, 26 Jan 2024 16:57:29 +0100 Subject: [PATCH 2/9] create base for powerbi calls --- backend/main.py | 2 ++ backend/requirements.txt | 3 ++- backend/routes/data_profile_routes.py | 2 +- backend/routes/powerbi_routes.py | 11 +++++++++++ backend/settings.py | 5 +++++ backend/utils/azure/azure_manager.py | 15 +++++++++++++++ backend/{ => utils}/object_storage/__init__.py | 0 .../object_storage/digitalocean_space_manager.py | 0 8 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 backend/routes/powerbi_routes.py create mode 100644 backend/utils/azure/azure_manager.py rename backend/{ => utils}/object_storage/__init__.py (100%) rename backend/{ => utils}/object_storage/digitalocean_space_manager.py (100%) diff --git a/backend/main.py b/backend/main.py index 13e0c35..0cb1784 100644 --- a/backend/main.py +++ b/backend/main.py @@ -9,6 +9,7 @@ from routes.data_profile_routes import data_profile_router from routes.file_routes import file_router from routes.organization_routes import organization_router +from routes.powerbi_routes import powerbi_router from routes.table_routes import table_router from routes.user_routes import user_router from settings import APP_ENV @@ -54,6 +55,7 @@ async def shutdown_event(): app.include_router(data_profile_router) app.include_router(file_router) app.include_router(organization_router) +app.include_router(powerbi_router) app.include_router(table_router) app.include_router(user_router) diff --git a/backend/requirements.txt b/backend/requirements.txt index fe8af01..00c1c41 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -23,4 +23,5 @@ sendgrid==6.11.0 boto3==1.34.10 pillow==10.1.0 pdf2image==1.16.3 -isort==5.13.2 \ No newline at end of file +isort==5.13.2 +azure-identity==1.15.0 \ No newline at end of file diff --git a/backend/routes/data_profile_routes.py b/backend/routes/data_profile_routes.py index 1a877c4..3721cab 100644 --- a/backend/routes/data_profile_routes.py +++ b/backend/routes/data_profile_routes.py @@ -14,9 +14,9 @@ DataProfileCreateResponse, ) from models.user import User -from object_storage.digitalocean_space_manager import DigitalOceanSpaceManager from security import get_current_user from utils.image_conversion_manager import ImageConversionManager +from utils.object_storage.digitalocean_space_manager import DigitalOceanSpaceManager data_profile_router = APIRouter() diff --git a/backend/routes/powerbi_routes.py b/backend/routes/powerbi_routes.py new file mode 100644 index 0000000..4bc09c7 --- /dev/null +++ b/backend/routes/powerbi_routes.py @@ -0,0 +1,11 @@ +from fastapi import APIRouter +from utils.azure.azure_manager import AzureManager + +powerbi_router = APIRouter() + + +@powerbi_router.get("/powerbi/token/") +async def get_powerbi_token(): + azure_manager = AzureManager() + token = azure_manager.get_powerbi_token() + return {"token": token} diff --git a/backend/settings.py b/backend/settings.py index 950c7b3..b3407fd 100644 --- a/backend/settings.py +++ b/backend/settings.py @@ -14,6 +14,11 @@ APP_ENV = config("APP_ENV") APP_HOST = config("APP_HOST") +AZURE_CLIENT_ID = config("AZURE_CLIENT_ID") +AZURE_TENANT_ID = config("AZURE_TENANT_ID") +AZURE_APP_VALUE = config("AZURE_APP_VALUE") +AZURE_APP_SECRET = config("AZURE_APP_SECRET") + ACCESS_TOKEN_EXPIRE_MINUTES = config("ACCESS_TOKEN_EXPIRE_MINUTES", default=30) REFRESH_TOKEN_EXPIRE_DAYS = config("REFRESH_TOKEN_EXPIRE_DAYS", default=1) REMEMBER_ME_ACCESS_TOKEN_EXPIRE_MINUTES = config( diff --git a/backend/utils/azure/azure_manager.py b/backend/utils/azure/azure_manager.py new file mode 100644 index 0000000..6cbe942 --- /dev/null +++ b/backend/utils/azure/azure_manager.py @@ -0,0 +1,15 @@ +from azure.identity import ClientSecretCredential +from settings import AZURE_APP_SECRET, AZURE_CLIENT_ID, AZURE_TENANT_ID + + +class AzureManager: + def __init__(self): + self.credential = ClientSecretCredential( + AZURE_TENANT_ID, AZURE_CLIENT_ID, AZURE_APP_SECRET + ) + self.powerbi_token = self.credential.get_token( + "https://analysis.windows.net/powerbi/api/.default" + ) + + def get_powerbi_token(self): + return self.powerbi_token.token diff --git a/backend/object_storage/__init__.py b/backend/utils/object_storage/__init__.py similarity index 100% rename from backend/object_storage/__init__.py rename to backend/utils/object_storage/__init__.py diff --git a/backend/object_storage/digitalocean_space_manager.py b/backend/utils/object_storage/digitalocean_space_manager.py similarity index 100% rename from backend/object_storage/digitalocean_space_manager.py rename to backend/utils/object_storage/digitalocean_space_manager.py From e58d983a49f365061977b36fec45f4082f45b35d Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Sat, 27 Jan 2024 13:04:33 +0100 Subject: [PATCH 3/9] begin work on suggesting data types --- backend/llms/system_message_manager.py | 4 ++++ .../pages/upload/CreateDataProfileWindow.jsx | 13 ++++++++++++- .../pages/upload/DataPreviewAndSchemaEditor.jsx | 17 ++++++++++------- 3 files changed, 26 insertions(+), 8 deletions(-) diff --git a/backend/llms/system_message_manager.py b/backend/llms/system_message_manager.py index e1793dc..5904570 100644 --- a/backend/llms/system_message_manager.py +++ b/backend/llms/system_message_manager.py @@ -4,6 +4,10 @@ def __init__(self): "analytics_chat": """ You are an analytics assistant. You will be generating SQL queries, and providing useful information for reports and analytics based on the given prompt.""", + "column_type_suggestion": """ + You are a column type suggestion assistant. + You will be suggesting PostgreSQL column types based on the given prompt. + """, "sql_code": """ You are a PostgreSQL SQL statement assistant. Generate PostgreSQL SQL statements based on the given prompt. diff --git a/frontend/src/pages/upload/CreateDataProfileWindow.jsx b/frontend/src/pages/upload/CreateDataProfileWindow.jsx index 3ed25ee..0f3a21e 100644 --- a/frontend/src/pages/upload/CreateDataProfileWindow.jsx +++ b/frontend/src/pages/upload/CreateDataProfileWindow.jsx @@ -55,7 +55,18 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { }; return ( - + { + onClose(); + setName(""); // Reset name + setExtractInstructions(""); // Reset extractInstructions + setSampleFiles([]); // Reset sampleFiles + setPreviewData(null); // Reset previewData + }} + maxWidth="md" + fullWidth + > Create a Data Profile
diff --git a/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx b/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx index 4146b15..82b72c4 100644 --- a/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx +++ b/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx @@ -26,7 +26,12 @@ function DataPreviewAndSchemaEditor({ previewData }) { useEffect(() => { if (data && data.length > 0) { const newColumnNames = Object.keys(data[0]); - const newColumnTypes = newColumnNames.map(() => "string"); + let newColumnTypes; + if (columnTypes.length === 0) { + newColumnTypes = newColumnNames.map(() => "text"); + } else { + newColumnTypes = columnTypes; + } if (JSON.stringify(newColumnNames) !== JSON.stringify(columnNames)) { setColumnNames(newColumnNames); } @@ -78,7 +83,7 @@ function DataPreviewAndSchemaEditor({ previewData }) { handleColumnTypeChange(index, event.target.value) } > - String + Text Number Boolean Date @@ -90,11 +95,9 @@ function DataPreviewAndSchemaEditor({ previewData }) { }; const handleColumnTypeChange = (index, newType) => { - setColumnTypes((prevColumnTypes) => { - const newColumnTypes = [...prevColumnTypes]; - newColumnTypes[index] = newType; - return newColumnTypes; - }); + let newColumnTypes = [...columnTypes]; + newColumnTypes[index] = newType; + setColumnTypes(newColumnTypes); }; const handleEditClick = (index) => { From 02cade8f7491052cc1be335cea038405dae88415 Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Sat, 27 Jan 2024 13:05:24 +0100 Subject: [PATCH 4/9] create endpoint for suggesting data types --- backend/llms/gpt.py | 11 +++++++++++ backend/routes/data_profile_routes.py | 8 ++++++++ 2 files changed, 19 insertions(+) diff --git a/backend/llms/gpt.py b/backend/llms/gpt.py index 4b03d4c..20d037e 100644 --- a/backend/llms/gpt.py +++ b/backend/llms/gpt.py @@ -368,6 +368,17 @@ async def generate_chart_config( return parsed_config + async def generate_suggested_column_types(self, data: dict): + """Generate suggested column types for the given data.""" + self._add_system_message(assistant_type="column_type_suggestion") + self._set_response_format(is_json=True) + + prompt = self.prompt_manager.create_column_type_suggestion_prompt(data) + + gpt_response = await self._send_and_receive_message(prompt) + + return gpt_response + def fetch_table_name_from_sample( self, sample_content: str, extra_desc: str, table_metadata: str ): diff --git a/backend/routes/data_profile_routes.py b/backend/routes/data_profile_routes.py index 3721cab..ea4f1f5 100644 --- a/backend/routes/data_profile_routes.py +++ b/backend/routes/data_profile_routes.py @@ -128,6 +128,14 @@ async def preview_data_profile( return extracted_data +# @data_profile_router.post("/data-profiles/preview/column-types/") +# async def generate_suggested_column_types( +# data, current_user: User = Depends(get_current_user) +# ): +# gpt = GPTLLM(chat_id=1, user=current_user) +# suggested_column_types = await gpt.generate_suggested_column_types(data) + + @data_profile_router.post("/data-profiles/{data_profile_name}/preview/") async def preview_data_profile_upload( data_profile_name: str, From df85c02f54ae1c69fc6e389ee8a453e239f4a2cd Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Mon, 29 Jan 2024 17:23:44 +0100 Subject: [PATCH 5/9] suggest data types for table columns --- backend/llms/gpt.py | 14 +- backend/llms/prompt_manager.py | 15 ++ backend/llms/system_message_manager.py | 2 +- backend/models/data_profile.py | 4 + backend/routes/data_profile_routes.py | 26 ++- frontend/src/api/dataProfilesRequests.jsx | 26 +++ .../pages/upload/CreateDataProfileWindow.jsx | 45 +++-- .../upload/DataPreviewAndSchemaEditor.jsx | 171 +++++++++--------- 8 files changed, 189 insertions(+), 114 deletions(-) create mode 100644 frontend/src/api/dataProfilesRequests.jsx diff --git a/backend/llms/gpt.py b/backend/llms/gpt.py index 20d037e..27f7445 100644 --- a/backend/llms/gpt.py +++ b/backend/llms/gpt.py @@ -368,16 +368,20 @@ async def generate_chart_config( return parsed_config - async def generate_suggested_column_types(self, data: dict): + async def generate_suggested_column_types(self, column_names: list, data: dict): """Generate suggested column types for the given data.""" self._add_system_message(assistant_type="column_type_suggestion") self._set_response_format(is_json=True) - prompt = self.prompt_manager.create_column_type_suggestion_prompt(data) + prompt = self.prompt_manager.create_column_type_suggestion_prompt( + column_names, data + ) gpt_response = await self._send_and_receive_message(prompt) - return gpt_response + suggested_column_types = json.loads(gpt_response) + + return suggested_column_types def fetch_table_name_from_sample( self, sample_content: str, extra_desc: str, table_metadata: str @@ -429,5 +433,9 @@ async def extract_data_from_jpgs( "\n```", "" ) data = json.loads(json_string) + + # If data is a dictionary, wrap it in a list + if isinstance(data, dict): + data = [data] print(data) return data diff --git a/backend/llms/prompt_manager.py b/backend/llms/prompt_manager.py index 61a1336..ef202a2 100644 --- a/backend/llms/prompt_manager.py +++ b/backend/llms/prompt_manager.py @@ -122,3 +122,18 @@ def jpg_data_extraction_prompt(self, instructions: str): Return only the requested information, no additional text or formatting. """ return prompt + + def create_column_type_suggestion_prompt(self, column_names, data): + prompt = f""" + Based on the following data, suggest the data types for each column in the table. + The available column types are: text, integer, money, date, boolean + + Column names: + {column_names} + + Data: + {data} + + Return a JSON with the column names as keys and the suggested data types as values. + """ + return prompt diff --git a/backend/llms/system_message_manager.py b/backend/llms/system_message_manager.py index 5904570..4a1f83b 100644 --- a/backend/llms/system_message_manager.py +++ b/backend/llms/system_message_manager.py @@ -6,7 +6,7 @@ def __init__(self): You will be generating SQL queries, and providing useful information for reports and analytics based on the given prompt.""", "column_type_suggestion": """ You are a column type suggestion assistant. - You will be suggesting PostgreSQL column types based on the given prompt. + You will be suggesting column data types based on the given prompt. """, "sql_code": """ You are a PostgreSQL SQL statement assistant. diff --git a/backend/models/data_profile.py b/backend/models/data_profile.py index b90d507..9e7dacf 100644 --- a/backend/models/data_profile.py +++ b/backend/models/data_profile.py @@ -52,3 +52,7 @@ class DataProfileCreateRequest(BaseModel): class DataProfileCreateResponse(BaseModel): name: str extract_instructions: str + + +class SuggestedColumnTypesRequest(BaseModel): + data: list diff --git a/backend/routes/data_profile_routes.py b/backend/routes/data_profile_routes.py index ea4f1f5..7a0ef7f 100644 --- a/backend/routes/data_profile_routes.py +++ b/backend/routes/data_profile_routes.py @@ -12,6 +12,7 @@ DataProfile, DataProfileCreateRequest, DataProfileCreateResponse, + SuggestedColumnTypesRequest, ) from models.user import User from security import get_current_user @@ -78,6 +79,11 @@ async def get_data_profile( return data_profile +@data_profile_router.get("/data-profiles/column-types/") +async def get_column_types(current_user: User = Depends(get_current_user)): + return ["text", "integer", "money", "date", "boolean"] + + @data_profile_router.post("/data-profiles/preview/") async def preview_data_profile( files: List[UploadFile] = File(...), @@ -128,12 +134,20 @@ async def preview_data_profile( return extracted_data -# @data_profile_router.post("/data-profiles/preview/column-types/") -# async def generate_suggested_column_types( -# data, current_user: User = Depends(get_current_user) -# ): -# gpt = GPTLLM(chat_id=1, user=current_user) -# suggested_column_types = await gpt.generate_suggested_column_types(data) +@data_profile_router.post("/data-profiles/preview/column-types/") +async def generate_suggested_column_types( + request: SuggestedColumnTypesRequest, current_user: User = Depends(get_current_user) +): + gpt = GPTLLM(chat_id=1, user=current_user) + if request.data: + column_names = list(request.data[0].keys()) + suggested_column_types = await gpt.generate_suggested_column_types( + column_names, request.data + ) + + print(suggested_column_types) + + return suggested_column_types @data_profile_router.post("/data-profiles/{data_profile_name}/preview/") diff --git a/frontend/src/api/dataProfilesRequests.jsx b/frontend/src/api/dataProfilesRequests.jsx new file mode 100644 index 0000000..e659d15 --- /dev/null +++ b/frontend/src/api/dataProfilesRequests.jsx @@ -0,0 +1,26 @@ +import axios from "axios"; +import { API_URL } from "../utils/constants"; + +export const getPreviewData = (sampleFiles, extractInstructions) => { + const formData = new FormData(); + sampleFiles.forEach((file) => { + formData.append("files", file); + }); + formData.append("extract_instructions", extractInstructions); + + return axios.post(`${API_URL}data-profiles/preview/`, formData, { + headers: { + "Content-Type": "multipart/form-data", + }, + }); +}; + +export const getAvailableColumnTypes = () => { + return axios.get(`${API_URL}data-profiles/column-types/`); +}; + +export const getSuggestedColumnTypes = (previewData) => { + return axios.post(`${API_URL}data-profiles/preview/column-types/`, { + data: previewData, + }); +}; diff --git a/frontend/src/pages/upload/CreateDataProfileWindow.jsx b/frontend/src/pages/upload/CreateDataProfileWindow.jsx index 0f3a21e..1cc9185 100644 --- a/frontend/src/pages/upload/CreateDataProfileWindow.jsx +++ b/frontend/src/pages/upload/CreateDataProfileWindow.jsx @@ -9,16 +9,21 @@ import { Stack, TextField, } from "@mui/material"; -import axios from "axios"; import FileUploader from "./FileUploader"; import DataPreviewAndSchemaEditor from "./DataPreviewAndSchemaEditor"; -import { API_URL } from "../../utils/constants"; +import { + getPreviewData, + getAvailableColumnTypes, + getSuggestedColumnTypes, +} from "../../api/dataProfilesRequests"; function CreateDataProfileWindow({ open, onClose, onCreate }) { const [name, setName] = useState(""); const [extractInstructions, setExtractInstructions] = useState(""); const [sampleFiles, setSampleFiles] = useState([]); const [previewData, setPreviewData] = useState(null); + const [availableColumnTypes, setAvailableColumnTypes] = useState([]); + const [selectedColumnTypes, setSelectedColumnTypes] = useState(null); const [isPreviewLoading, setIsPreviewLoading] = useState(false); const [isPreviewTableOpen, setIsPreviewTableOpen] = useState(false); @@ -30,25 +35,33 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { const handlePreview = () => { if (sampleFiles.length && extractInstructions) { setIsPreviewLoading(true); + setPreviewData(null); + setSelectedColumnTypes(null); + const formData = new FormData(); sampleFiles.forEach((file) => { - formData.append("files", file); // Append each file + formData.append("files", file); }); formData.append("extract_instructions", extractInstructions); - axios - .post(`${API_URL}data-profiles/preview/`, formData, { - headers: { - "Content-Type": "multipart/form-data", - }, + Promise.all([ + getPreviewData(sampleFiles, extractInstructions), + getAvailableColumnTypes(), + ]) + .then(([previewDataResponse, availableTypesResponse]) => { + setPreviewData(previewDataResponse.data); + setAvailableColumnTypes(availableTypesResponse.data); + + return getSuggestedColumnTypes(previewDataResponse.data); }) - .then((response) => { - setPreviewData(response.data); // Store the preview data + .then((suggestedTypesResponse) => { + setSelectedColumnTypes(suggestedTypesResponse.data); setIsPreviewTableOpen(true); - setIsPreviewLoading(false); }) .catch((error) => { - console.error("Error on preview:", error); + console.error("Error during preview setup:", error); + }) + .finally(() => { setIsPreviewLoading(false); }); } @@ -95,8 +108,12 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { /> - {previewData && ( - + {previewData && selectedColumnTypes && ( + )} diff --git a/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx b/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx index 82b72c4..7b16231 100644 --- a/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx +++ b/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx @@ -1,4 +1,4 @@ -import React, { useState, useEffect, useRef } from "react"; +import React, { useState, useEffect } from "react"; import { Box, IconButton, @@ -13,117 +13,108 @@ import { TableHead, TableRow, TextField, + Tooltip, } from "@mui/material"; import EditIcon from "@mui/icons-material/Edit"; -function DataPreviewAndSchemaEditor({ previewData }) { - const data = Array.isArray(previewData) ? previewData : [previewData]; - const [columnNames, setColumnNames] = useState([]); - const [columnTypes, setColumnTypes] = useState([]); - const [editingColumnIndex, setEditingColumnIndex] = useState(null); - const inputRefs = useRef([]); +function DataPreviewAndSchemaEditor({ + previewData, + availableColumnTypes, + selectedColumnTypes, +}) { + const [columns, setColumns] = useState([]); useEffect(() => { - if (data && data.length > 0) { - const newColumnNames = Object.keys(data[0]); - let newColumnTypes; - if (columnTypes.length === 0) { - newColumnTypes = newColumnNames.map(() => "text"); - } else { - newColumnTypes = columnTypes; - } - if (JSON.stringify(newColumnNames) !== JSON.stringify(columnNames)) { - setColumnNames(newColumnNames); - } - if (JSON.stringify(newColumnTypes) !== JSON.stringify(columnTypes)) { - setColumnTypes(newColumnTypes); - } + if (Array.isArray(previewData) && previewData.length > 0) { + const initialColumns = Object.keys(previewData[0]).map((key) => ({ + name: key, + type: selectedColumnTypes[key] || "text", + isEditing: false, + })); + setColumns(initialColumns); } - }, [data]); - - const generateHeaderRow = (data) => { - if (data && data.length > 0) { - return columnNames.map((key, index) => ( - - - handleColumnNameChange(index, event.target.value) - } - variant="standard" - InputProps={{ - disableUnderline: true, - readOnly: editingColumnIndex !== index, - endAdornment: ( - - handleEditClick(index)}> - - - - ), - }} - inputRef={(ref) => (inputRefs.current[index] = ref)} - onClick={() => handleEditClick(index)} - style={{ cursor: "pointer" }} - inputProps={{ - style: { - cursor: editingColumnIndex === index ? "text" : "pointer", - }, - }} - onKeyDown={(event) => { - if (event.key === "Enter") { - event.preventDefault(); - setEditingColumnIndex(null); - } - }} - /> - - - )); - } - }; + }, [previewData, selectedColumnTypes]); const handleColumnTypeChange = (index, newType) => { - let newColumnTypes = [...columnTypes]; - newColumnTypes[index] = newType; - setColumnTypes(newColumnTypes); + setColumns((prevColumns) => + prevColumns.map((column, colIndex) => + colIndex === index ? { ...column, type: newType } : column, + ), + ); }; const handleEditClick = (index) => { - setEditingColumnIndex(index); - inputRefs.current[index].select(); + setColumns((prevColumns) => + prevColumns.map((column, colIndex) => + colIndex === index + ? { ...column, isEditing: !column.isEditing } + : column, + ), + ); }; const handleColumnNameChange = (index, newName) => { - setColumnNames((prevColumnNames) => { - const newColumnNames = [...prevColumnNames]; - newColumnNames[index] = newName; - return newColumnNames; - }); + setColumns((prevColumns) => + prevColumns.map((column, colIndex) => + colIndex === index ? { ...column, name: newName } : column, + ), + ); }; return ( - {generateHeaderRow(data)} + + {columns.map((column, index) => ( + + + handleColumnNameChange(index, event.target.value) + } + variant="standard" + InputProps={{ + disableUnderline: true, + readOnly: !column.isEditing, + endAdornment: ( + + handleEditClick(index)}> + + + + ), + }} + style={{ cursor: "pointer" }} + /> + + + + + ))} + - {data.map((row, index) => ( - - {Object.values(row).map((value, idx) => ( - {value} + {previewData.map((row, rowIndex) => ( + + {Object.values(row).map((value, cellIndex) => ( + {value} ))} ))} From 090ceb8cac910f23f13f51a4dedc9e3b44b32d6f Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Mon, 29 Jan 2024 17:49:07 +0100 Subject: [PATCH 6/9] specify version of black --- .github/workflows/python-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml index 2292f67..7833258 100644 --- a/.github/workflows/python-quality.yml +++ b/.github/workflows/python-quality.yml @@ -18,7 +18,7 @@ jobs: python-version: '3.8' - name: Install dependencies - run: pip install black flake8 mypy isort + run: pip install black==23.11.10 flake8 mypy isort - name: Check Python code formatting with Black run: black --check --exclude backend/alembic/versions backend/ From 78593337d2c43af0445204005b692d58f0c710d1 Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Mon, 29 Jan 2024 17:50:41 +0100 Subject: [PATCH 7/9] specify version of black --- .github/workflows/python-quality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-quality.yml b/.github/workflows/python-quality.yml index 7833258..354be6e 100644 --- a/.github/workflows/python-quality.yml +++ b/.github/workflows/python-quality.yml @@ -18,7 +18,7 @@ jobs: python-version: '3.8' - name: Install dependencies - run: pip install black==23.11.10 flake8 mypy isort + run: pip install black==23.11.0 flake8 mypy isort - name: Check Python code formatting with Black run: black --check --exclude backend/alembic/versions backend/ From 88f6f2ce715d37ef709da91eaa195daa60050d40 Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Mon, 29 Jan 2024 18:01:43 +0100 Subject: [PATCH 8/9] make whole file uploader clickable --- frontend/src/pages/upload/FileUploader.jsx | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/frontend/src/pages/upload/FileUploader.jsx b/frontend/src/pages/upload/FileUploader.jsx index 2258f17..5abef76 100644 --- a/frontend/src/pages/upload/FileUploader.jsx +++ b/frontend/src/pages/upload/FileUploader.jsx @@ -1,10 +1,11 @@ -import React, { useState } from "react"; +import React, { useRef, useState } from "react"; import { Box, Typography, IconButton } from "@mui/material"; import DeleteIcon from "@mui/icons-material/Delete"; const FileUploader = ({ setFiles, id }) => { const [fileNames, setFileNames] = useState([]); const [fileType, setFileType] = useState(null); + const fileInputRef = useRef(null); const handleFileChange = (event) => { const uploadedFiles = Array.from(event.target.files); @@ -57,7 +58,8 @@ const FileUploader = ({ setFiles, id }) => { } }; - const handleDelete = (indexToDelete) => { + const handleDelete = (event, indexToDelete) => { + event.stopPropagation(); setFileNames(fileNames.filter((_, index) => index !== indexToDelete)); setFiles((prevFiles) => Array.from(prevFiles).filter((_, index) => index !== indexToDelete), @@ -67,6 +69,10 @@ const FileUploader = ({ setFiles, id }) => { } }; + const handleClickBox = () => { + fileInputRef.current.click(); + }; + return ( { }} onDragOver={handleDragOver} onDrop={handleDrop} + onClick={handleClickBox} > { style={{ display: "none" }} onChange={handleFileChange} multiple + ref={fileInputRef} /> -