From 93885204df8bebecdf1f8afa1daf7659a148253e Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Tue, 6 Feb 2024 23:56:06 +0100 Subject: [PATCH 1/2] remove unused data-profiling directory --- .../data-profiling/CreateDataProfile.jsx | 59 ----- .../data-profiling/DataProfilingPage.jsx | 228 ------------------ .../SpecificDataProfilePage.jsx | 85 ------- 3 files changed, 372 deletions(-) delete mode 100644 frontend/src/pages/data-profiling/CreateDataProfile.jsx delete mode 100644 frontend/src/pages/data-profiling/DataProfilingPage.jsx delete mode 100644 frontend/src/pages/data-profiling/SpecificDataProfilePage.jsx diff --git a/frontend/src/pages/data-profiling/CreateDataProfile.jsx b/frontend/src/pages/data-profiling/CreateDataProfile.jsx deleted file mode 100644 index a8f33dc..0000000 --- a/frontend/src/pages/data-profiling/CreateDataProfile.jsx +++ /dev/null @@ -1,59 +0,0 @@ -import React, { useState } from "react"; -import { Box, TextField, Button, Typography } from "@mui/material"; -import axios from "axios"; -import { useNavigate } from "react-router-dom"; -import { API_URL } from "../../utils/constants"; - -const CreateDataProfile = () => { - const [name, setName] = useState(""); - const [description, setDescription] = useState(""); - const navigate = useNavigate(); - - const handleSubmit = (event) => { - event.preventDefault(); - axios - .post(`${API_URL}data-profile/`, { name, description }) - .then((response) => { - // Handle successful data profile creation - console.log("Data Profile created:", response.data); - navigate("/data-profiling"); - }) - .catch((error) => { - console.error("Error creating data profile:", error); - }); - }; - - const handleBack = () => { - navigate("/data-profiling"); - }; - - return ( - - Create New Data Profile - setName(e.target.value)} - margin="normal" - /> - setDescription(e.target.value)} - margin="normal" - /> - - - - ); -}; - -export default CreateDataProfile; diff --git a/frontend/src/pages/data-profiling/DataProfilingPage.jsx b/frontend/src/pages/data-profiling/DataProfilingPage.jsx deleted file mode 100644 index 9f29eca..0000000 --- a/frontend/src/pages/data-profiling/DataProfilingPage.jsx +++ /dev/null @@ -1,228 +0,0 @@ -import React, { useState, useEffect, useRef } from "react"; -import axios from "axios"; -import { - Box, - Typography, - Table, - TableBody, - TableCell, - TableContainer, - TableHead, - TableRow, - Paper, - Link, - Button, - TextField, -} from "@mui/material"; -import { useNavigate } from "react-router-dom"; -import { API_URL } from "../../utils/constants"; - -function DataProfilingPage() { - const [dataProfiles, setDataProfiles] = useState([]); - const [instructions, setInstructions] = useState(""); - const [isUploading, setIsUploading] = useState(false); - const [previewData, setPreviewData] = useState(null); - const navigate = useNavigate(); - const fileInputRef = useRef(null); - const [selectedFiles, setSelectedFiles] = useState([]); // Array of files - const [selectedFileNames, setSelectedFileNames] = useState([]); // Array of file names - - useEffect(() => { - axios - .get(`${API_URL}data-profiles/`) - .then((response) => { - if (Array.isArray(response.data)) { - setDataProfiles(response.data); - } else { - console.error("Received data is not an array:", response.data); - setDataProfiles([]); - } - }) - .catch((error) => console.error("Error fetching data profiles:", error)); - }, []); - - const generateTableHeaders = (data) => { - if (data && data.length > 0) { - return Object.keys(data[0]).map((key) => ( - {key.replace(/_/g, " ").toUpperCase()} - )); - } - return null; - }; - - const handleProfileCreate = () => { - navigate("/data-profiling/create"); - }; - - const handleUploadClick = () => { - fileInputRef.current.click(); - }; - - const handleFileChange = (event) => { - const files = Array.from(event.target.files); - if (files.length) { - setSelectedFiles(files); - setSelectedFileNames(files.map((file) => file.name)); - } - }; - - const handleInstructionsChange = (event) => { - setInstructions(event.target.value); - }; - - const handleUpload = () => { - if (selectedFiles.length) { - setIsUploading(true); - const formData = new FormData(); - selectedFiles.forEach((file) => { - formData.append("files", file); // Append each file to the form data - }); - formData.append("instructions", instructions); - - axios - .post(`${API_URL}upload-url`, formData) - .then((response) => { - console.log(response); - setIsUploading(false); - // Reset states after upload - setSelectedFiles([]); - setSelectedFileNames([]); - setInstructions(""); - }) - .catch((error) => { - console.error("Error uploading file:", error); - setIsUploading(false); - }); - } - }; - - const handlePreview = () => { - if (selectedFiles.length && instructions) { - const formData = new FormData(); - selectedFiles.forEach((file) => { - formData.append("files", file); // Append each file - }); - formData.append("instructions", instructions); - - axios - .post(`${API_URL}data-profiles/preview/`, formData, { - headers: { - "Content-Type": "multipart/form-data", - }, - }) - .then((response) => { - setPreviewData(response.data); // Store the preview data - }) - .catch((error) => console.error("Error on preview:", error)); - } - }; - - return ( - - - 🔍 Data Profiling - - - - - - - - - {/* Hidden File Input */} - - - - - - {selectedFileNames.length > 0 && ( - - Files selected: {selectedFileNames.join(", ")} - - )} - - {/* Display preview data if available */} - {previewData && ( - - - - {generateTableHeaders(previewData)} - - - {previewData.map((row, index) => ( - - {Object.values(row).map((value, idx) => ( - {value} - ))} - - ))} - -
- - )} - - - - - - Name - Description - - - - {dataProfiles.map((profile) => ( - - - handleProfileClick(profile.id)} - > - {profile.name} - - - {profile.description} - - ))} - -
- - - ); -} - -export default DataProfilingPage; diff --git a/frontend/src/pages/data-profiling/SpecificDataProfilePage.jsx b/frontend/src/pages/data-profiling/SpecificDataProfilePage.jsx deleted file mode 100644 index 9ecfbab..0000000 --- a/frontend/src/pages/data-profiling/SpecificDataProfilePage.jsx +++ /dev/null @@ -1,85 +0,0 @@ -import React, { useState, useEffect } from "react"; -import { useParams } from "react-router-dom"; -import axios from "axios"; -import { - Box, - Typography, - CircularProgress, - Table, - TableBody, - TableCell, - TableRow, - TableContainer, - Paper, -} from "@mui/material"; -import { API_URL } from "../../utils/constants"; - -function SpecificDataProfilePage() { - const { dataProfileId } = useParams(); - const [profile, setProfile] = useState(null); - const [loading, setLoading] = useState(true); - const [error, setError] = useState(null); - - useEffect(() => { - axios - .get(`${API_URL}/data-profiles/${dataProfileId}`) - .then((response) => { - setProfile(response.data); - setLoading(false); - }) - .catch((error) => { - console.error("Error fetching data profile:", error); - setError(error); - setLoading(false); - }); - }, [dataProfileId]); - - if (loading) { - return ; - } - - if (error) { - return ( - - Error loading profile - - ); - } - - return ( - - - Data Profile Details - - - - - - ID - {profile.id} - - - Name - {profile.name} - - - File Type - {profile.file_type} - - - Organization ID - {profile.organization_id} - - - Description - {profile.description} - - {/* Add more rows as needed */} - -
- - - ); -} - -export default SpecificDataProfilePage; From 009b3ad0041d1e3e10b909008b8a9a5b9c3672cc Mon Sep 17 00:00:00 2001 From: liberty-rising Date: Mon, 12 Feb 2024 19:19:24 +0100 Subject: [PATCH 2/2] implement ability for using a primary in creation of dataprofiles --- backend/database/sql_executor.py | 4 +- backend/database/table_manager.py | 6 +- backend/llms/gpt.py | 6 +- backend/llms/prompt_manager.py | 19 ++- backend/llms/system_message_manager.py | 6 +- backend/models/data_profile.py | 15 +- backend/routes/data_profile_routes.py | 29 +++- backend/utils/sql_string_manager.py | 16 +- frontend/src/api/dataProfilesRequests.jsx | 4 +- frontend/src/components/tabs/TabPanel.jsx | 24 +++ frontend/src/pages/login/LoginPage.jsx | 2 +- .../pages/upload/CreateDataProfileWindow.jsx | 33 ++-- .../pages/upload/DataFormatEditorTable.jsx | 155 ++++++++++++++++++ .../upload/DataPreviewAndFormatEditor.jsx | 59 +++++++ .../upload/DataPreviewAndSchemaEditor.jsx | 132 --------------- ...{PreviewTable.jsx => DataPreviewTable.jsx} | 22 ++- frontend/src/pages/upload/UploadPage.jsx | 11 +- frontend/src/styles/tableStyles.css | 4 + 18 files changed, 350 insertions(+), 197 deletions(-) create mode 100644 frontend/src/components/tabs/TabPanel.jsx create mode 100644 frontend/src/pages/upload/DataFormatEditorTable.jsx create mode 100644 frontend/src/pages/upload/DataPreviewAndFormatEditor.jsx delete mode 100644 frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx rename frontend/src/pages/upload/{PreviewTable.jsx => DataPreviewTable.jsx} (81%) diff --git a/backend/database/sql_executor.py b/backend/database/sql_executor.py index 558cfc5..5c34d8f 100644 --- a/backend/database/sql_executor.py +++ b/backend/database/sql_executor.py @@ -23,13 +23,13 @@ def append_df_to_table(self, df: pd.DataFrame, table_name: str): raise def create_table_for_data_profile( - self, org_id: int, table_name: str, column_names_and_types: dict + self, org_id: int, table_name: str, column_metadata: dict ): """Creates a table for a data profile.""" try: create_query = ( self.sql_string_manager.generate_create_query_for_data_profile_table( - table_name, column_names_and_types + table_name, column_metadata ) ) self.session.execute(text(create_query)) diff --git a/backend/database/table_manager.py b/backend/database/table_manager.py index 9d846c6..4b860aa 100644 --- a/backend/database/table_manager.py +++ b/backend/database/table_manager.py @@ -57,14 +57,12 @@ def create_table_for_data_profile( org_id: int, table_name: str, table_alias: str, - column_names_and_types: dict, + column_metadata: dict, ): """Creates a table for a data profile.""" try: executor = SQLExecutor(self.session) - executor.create_table_for_data_profile( - org_id, table_name, column_names_and_types - ) + executor.create_table_for_data_profile(org_id, table_name, column_metadata) self._map_table_to_org(org_id, table_name, table_alias) except Exception as e: print(f"An error occurred: {e}") diff --git a/backend/llms/gpt.py b/backend/llms/gpt.py index 27f7445..00e726d 100644 --- a/backend/llms/gpt.py +++ b/backend/llms/gpt.py @@ -368,12 +368,12 @@ async def generate_chart_config( return parsed_config - async def generate_suggested_column_types(self, column_names: list, data: dict): + async def generate_suggested_column_metadata(self, column_names: list, data: dict): """Generate suggested column types for the given data.""" - self._add_system_message(assistant_type="column_type_suggestion") + self._add_system_message(assistant_type="column_metadata_suggestion") self._set_response_format(is_json=True) - prompt = self.prompt_manager.create_column_type_suggestion_prompt( + prompt = self.prompt_manager.create_column_metadata_suggestion_prompt( column_names, data ) diff --git a/backend/llms/prompt_manager.py b/backend/llms/prompt_manager.py index ef202a2..c138002 100644 --- a/backend/llms/prompt_manager.py +++ b/backend/llms/prompt_manager.py @@ -123,10 +123,10 @@ def jpg_data_extraction_prompt(self, instructions: str): """ return prompt - def create_column_type_suggestion_prompt(self, column_names, data): + def create_column_metadata_suggestion_prompt(self, column_names, data): prompt = f""" - Based on the following data, suggest the data types for each column in the table. - The available column types are: text, integer, money, date, boolean + Based on the following data, suggest the data types for each column in the table and indicate which column should be a primary key. + The available data types are: text, integer, money, date, boolean. Column names: {column_names} @@ -134,6 +134,17 @@ def create_column_type_suggestion_prompt(self, column_names, data): Data: {data} - Return a JSON with the column names as keys and the suggested data types as values. + Return a JSON object where each key is a column name. + For each key, provide an object specifying 'data_type' and 'primary_key' status (a boolean indicating whether the column is a primary key). + + Example output: + {{ + client_name: {{ data_type: "text", primary_key: false }}, + net_amount: {{ data_type: "money", primary_key: true }}, + gross_amount: {{ data_type: "money", primary_key: false }}, + date: {{ data_type: "date", primary_key: false }}, + }} + + If no column appears that it should be a primary key, set the 'primary_key' value to false for all columns. """ return prompt diff --git a/backend/llms/system_message_manager.py b/backend/llms/system_message_manager.py index 4a1f83b..7e56f18 100644 --- a/backend/llms/system_message_manager.py +++ b/backend/llms/system_message_manager.py @@ -4,9 +4,9 @@ def __init__(self): "analytics_chat": """ You are an analytics assistant. You will be generating SQL queries, and providing useful information for reports and analytics based on the given prompt.""", - "column_type_suggestion": """ - You are a column type suggestion assistant. - You will be suggesting column data types based on the given prompt. + "column_metadata_suggestion": """ + You are a column metadata suggestion assistant. + You will be suggesting column data types and primary keys based on the given prompt. """, "sql_code": """ You are a PostgreSQL SQL statement assistant. diff --git a/backend/models/data_profile.py b/backend/models/data_profile.py index 0e8fdd0..0244395 100644 --- a/backend/models/data_profile.py +++ b/backend/models/data_profile.py @@ -1,3 +1,5 @@ +from typing import Dict, Union + from pydantic import BaseModel from sqlalchemy import Column, ForeignKey, Integer, String, UniqueConstraint @@ -45,9 +47,20 @@ def to_dict(self): class DataProfileCreateRequest(BaseModel): + """ + DataProfileCreateRequest Model + ------------------------------ + This class represents the request body for creating a new data profile. + Attributes: + - name: The name of the data profile. + - extract_instructions: The instructions for extracting data from the file. + - column_metadata: A dictionary where each key is a column name and each value is another dictionary specifying the attributes of the column. + The inner dictionary includes 'data_type' and 'primary_key' fields. + """ + name: str extract_instructions: str - column_names_and_types: dict + column_metadata: Dict[str, Dict[str, Union[str, bool]]] class DataProfileCreateResponse(BaseModel): diff --git a/backend/routes/data_profile_routes.py b/backend/routes/data_profile_routes.py index 6c3331e..fa0caff 100644 --- a/backend/routes/data_profile_routes.py +++ b/backend/routes/data_profile_routes.py @@ -45,7 +45,25 @@ async def get_data_profiles_by_org_id(current_user: User = Depends(get_current_u async def save_data_profile( request: DataProfileCreateRequest, current_user: User = Depends(get_current_user) ) -> DataProfileCreateResponse: - """Save a new data profile to the database""" + """ + Creates a new data profile and saves it to the database. + + This function first validates the name of the data profile, ensuring it is not longer than 50 characters and only contains valid characters for a table name. + It then checks if a data profile with the same name already exists for the current user's organization. + + If the validation passes and no duplicate data profile exists, it creates a new table for the data profile using the provided column metadata. + It then creates a new data profile with the provided name, extract instructions, and the current user's organization id, and saves it to the database. + + Args: + request (DataProfileCreateRequest): The data profile creation request containing the name, extract instructions, and column metadata for the new data profile. + current_user (User, optional): The current user. Defaults to the result of `get_current_user()`. + + Raises: + HTTPException: If the data profile name is invalid or a data profile with the same name already exists for the current user's organization. + + Returns: + DataProfileCreateResponse: The created data profile. + """ if len(request.name) > 50: raise HTTPException( status_code=400, detail="Data Profile name cannot be longer than 50 chars" @@ -73,7 +91,7 @@ async def save_data_profile( org_id=current_user.organization_id, table_name=table_name, table_alias=request.name, - column_names_and_types=request.column_names_and_types, + column_metadata=request.column_metadata, ) # Create the data profile @@ -160,14 +178,14 @@ async def preview_data_profile( return extracted_data -@data_profile_router.post("/data-profiles/preview/column-types/") -async def generate_suggested_column_types( +@data_profile_router.post("/data-profiles/preview/column-metadata/") +async def generate_suggested_column_metadata( request: SuggestedColumnTypesRequest, current_user: User = Depends(get_current_user) ): gpt = GPTLLM(chat_id=1, user=current_user) if request.data: column_names = list(request.data[0].keys()) - suggested_column_types = await gpt.generate_suggested_column_types( + suggested_column_types = await gpt.generate_suggested_column_metadata( column_names, request.data ) @@ -248,6 +266,7 @@ async def save_extracted_data( files: List[UploadFile] = File(...), current_user: User = Depends(get_current_user), ): + """Save the extracted data to the database using the data profile. Save the original files to DigitalOcean Spaces.""" # Get the organization name with DatabaseManager() as session: org_manager = OrganizationManager(session) diff --git a/backend/utils/sql_string_manager.py b/backend/utils/sql_string_manager.py index a7f1ea2..cd16e48 100644 --- a/backend/utils/sql_string_manager.py +++ b/backend/utils/sql_string_manager.py @@ -52,24 +52,30 @@ def map_to_postgres_type(self, column_type: str) -> str: return type_mapping.get(column_type, "TEXT") def generate_create_query_for_data_profile_table( - self, table_name: str, column_names_and_types: dict + self, table_name: str, column_metadata: dict ) -> str: """ Generates a CREATE TABLE query for a data profile table. Parameters: table_name (str): The name of the table. - column_names_and_types (dict): A dictionary of column names and types. + column_metadata (dict): A dictionary of column names, types, and primary key information. Returns: str: The CREATE TABLE query. """ # Generate the CREATE TABLE query create_query = f"CREATE TABLE {table_name} (" - for column_name, column_type in column_names_and_types.items(): - postgres_type = self.map_to_postgres_type(column_type) + primary_key = None + for column_name, column_info in column_metadata.items(): + postgres_type = self.map_to_postgres_type(column_info["data_type"]) create_query += f"{column_name} {postgres_type}, " - create_query = create_query[:-2] + ");" + if column_info.get("primary_key"): + primary_key = column_name + create_query = create_query[:-2] + if primary_key: + create_query += f", PRIMARY KEY ({primary_key})" + create_query += ");" return create_query diff --git a/frontend/src/api/dataProfilesRequests.jsx b/frontend/src/api/dataProfilesRequests.jsx index e659d15..d0c1fe6 100644 --- a/frontend/src/api/dataProfilesRequests.jsx +++ b/frontend/src/api/dataProfilesRequests.jsx @@ -19,8 +19,8 @@ export const getAvailableColumnTypes = () => { return axios.get(`${API_URL}data-profiles/column-types/`); }; -export const getSuggestedColumnTypes = (previewData) => { - return axios.post(`${API_URL}data-profiles/preview/column-types/`, { +export const getSuggestedColumnMetadata = (previewData) => { + return axios.post(`${API_URL}data-profiles/preview/column-metadata/`, { data: previewData, }); }; diff --git a/frontend/src/components/tabs/TabPanel.jsx b/frontend/src/components/tabs/TabPanel.jsx new file mode 100644 index 0000000..a7e4047 --- /dev/null +++ b/frontend/src/components/tabs/TabPanel.jsx @@ -0,0 +1,24 @@ +import Box from "@mui/material/Box"; +import Typography from "@mui/material/Typography"; + +function TabPanel(props) { + const { children, value, index, ...other } = props; + + return ( + + ); +} + +export default TabPanel; diff --git a/frontend/src/pages/login/LoginPage.jsx b/frontend/src/pages/login/LoginPage.jsx index 6de3887..b6ae372 100644 --- a/frontend/src/pages/login/LoginPage.jsx +++ b/frontend/src/pages/login/LoginPage.jsx @@ -184,7 +184,7 @@ function LoginPage({ onLogin }) { align="center" sx={{ mt: 5 }} > - Copyright © Your Website 2023. + Copyright © DocShow AI 2024. diff --git a/frontend/src/pages/upload/CreateDataProfileWindow.jsx b/frontend/src/pages/upload/CreateDataProfileWindow.jsx index db2e677..b422dda 100644 --- a/frontend/src/pages/upload/CreateDataProfileWindow.jsx +++ b/frontend/src/pages/upload/CreateDataProfileWindow.jsx @@ -10,11 +10,11 @@ import { TextField, } from "@mui/material"; import FileUploader from "./FileUploader"; -import DataPreviewAndSchemaEditor from "./DataPreviewAndSchemaEditor"; +import DataPreviewAndFormatEditor from "./DataPreviewAndFormatEditor"; import { getPreviewData, getAvailableColumnTypes, - getSuggestedColumnTypes, + getSuggestedColumnMetadata, } from "../../api/dataProfilesRequests"; function CreateDataProfileWindow({ open, onClose, onCreate }) { @@ -23,21 +23,20 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { const [sampleFiles, setSampleFiles] = useState([]); const [previewData, setPreviewData] = useState(null); const [availableColumnTypes, setAvailableColumnTypes] = useState([]); - const [selectedColumnTypes, setSelectedColumnTypes] = useState(null); + const [columnMetadata, setColumnMetadata] = useState(null); const [isPreviewLoading, setIsPreviewLoading] = useState(false); const [isPreviewTableOpen, setIsPreviewTableOpen] = useState(false); - const [columnNamesAndTypes, setColumnNamesAndTypes] = useState({}); const handleSubmit = (event) => { event.preventDefault(); - onCreate(name, extractInstructions, columnNamesAndTypes); + onCreate(name, extractInstructions, columnMetadata); }; const handlePreview = () => { if (sampleFiles.length && extractInstructions) { setIsPreviewLoading(true); setPreviewData(null); - setSelectedColumnTypes(null); + setColumnMetadata(null); const formData = new FormData(); sampleFiles.forEach((file) => { @@ -53,10 +52,10 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { setPreviewData(previewDataResponse.data); setAvailableColumnTypes(availableTypesResponse.data); - return getSuggestedColumnTypes(previewDataResponse.data); + return getSuggestedColumnMetadata(previewDataResponse.data); }) .then((suggestedTypesResponse) => { - setSelectedColumnTypes(suggestedTypesResponse.data); + setColumnMetadata(suggestedTypesResponse.data); setIsPreviewTableOpen(true); }) .catch((error) => { @@ -68,15 +67,6 @@ function CreateDataProfileWindow({ open, onClose, onCreate }) { } }; - const handleColumnsChange = (columns) => { - const newColumnNamesAndTypes = columns.reduce((acc, column) => { - acc[column.name] = column.type; - return acc; - }, {}); - - setColumnNamesAndTypes(newColumnNamesAndTypes); - }; - return ( - {previewData && selectedColumnTypes && ( - )} diff --git a/frontend/src/pages/upload/DataFormatEditorTable.jsx b/frontend/src/pages/upload/DataFormatEditorTable.jsx new file mode 100644 index 0000000..c1e6eed --- /dev/null +++ b/frontend/src/pages/upload/DataFormatEditorTable.jsx @@ -0,0 +1,155 @@ +import React, { useEffect, useState } from "react"; +import { Column } from "primereact/column"; +import { Dropdown } from "primereact/dropdown"; +import { DataTable } from "primereact/datatable"; +import { InputText } from "primereact/inputtext"; +import { RadioButton } from "primereact/radiobutton"; + +function DataFormatEditorTable({ + previewData, + setPreviewData, + availableColumnTypes, + columnMetadata, + setColumnMetadata, + primaryKey, + setPrimaryKey, +}) { + const EditableInputText = ({ + columnName, + previewData, + setPreviewData, + columnMetadata, + setColumnMetadata, + }) => { + const [editedColumnName, setEditedColumnName] = useState( + columnName.replace(/_/g, " ").toUpperCase(), + ); + + return ( + { + setEditedColumnName(e.target.value); + }} + onKeyDown={(e) => { + if (e.key === "Enter") { + e.target.blur(); + } + }} + onFocus={(e) => { + e.target.select(); + }} + onBlur={(e) => { + const newColumnName = e.target.value.replace(/ /g, "_").toLowerCase(); + + if (newColumnName !== columnName) { + const newPreviewData = previewData.map((row) => { + const newRow = Object.keys(row).reduce((acc, key) => { + if (key === columnName) { + acc[newColumnName] = row[key]; + } else { + acc[key] = row[key]; + } + return acc; + }, {}); + + return newRow; + }); + + // Update the previewData state + setPreviewData(newPreviewData); + + // Update the columnMetadata state if column name is changed + const newColumnMetadata = { ...columnMetadata }; + newColumnMetadata[newColumnName] = newColumnMetadata[columnName]; + if ( + newColumnMetadata[columnName] && + newColumnMetadata[columnName].primary_key + ) { + newColumnMetadata[newColumnName].primary_key = true; + delete newColumnMetadata[columnName]; + } + setColumnMetadata(newColumnMetadata); + + // Reset the editedColumnName state + setEditedColumnName(newColumnName.replace(/_/g, " ").toUpperCase()); + } + }} + /> + ); + }; + return ( + ({ + column: ( + + ), + primaryKey: ( + { + // Create a copy of the columnMetadata state + const newColumnMetadata = { ...columnMetadata }; + + // Iterate over the newColumnMetadata object + for (let key in newColumnMetadata) { + // If the key matches the new primary key, set its primary_key attribute to true + if (key === columnName) { + newColumnMetadata[key].primary_key = true; + } + // Otherwise, set the primary_key attribute to false + else { + newColumnMetadata[key].primary_key = false; + } + } + + // Update the columnMetadata state + setColumnMetadata(newColumnMetadata); + }} + /> + ), + type: ( + ({ + label: option.charAt(0).toUpperCase() + option.slice(1), + value: option, + }))} + filter + onChange={(e) => { + // Update the columnMetadata state here + setColumnMetadata({ + ...columnMetadata, + [columnName]: e.value, + }); + }} + style={{ width: "100%" }} + /> + ), + }))} + paginator + rows={5} + rowsPerPageOptions={[5, 10, 25, 50]} + paginatorTemplate="RowsPerPageDropdown FirstPageLink PrevPageLink CurrentPageReport NextPageLink LastPageLink" + currentPageReportTemplate="{first}-{last} of {totalRecords}" + showGridlines + resizableColumns + > + + + + + ); +} + +export default DataFormatEditorTable; diff --git a/frontend/src/pages/upload/DataPreviewAndFormatEditor.jsx b/frontend/src/pages/upload/DataPreviewAndFormatEditor.jsx new file mode 100644 index 0000000..c7453c1 --- /dev/null +++ b/frontend/src/pages/upload/DataPreviewAndFormatEditor.jsx @@ -0,0 +1,59 @@ +import React, { useState, useEffect } from "react"; +import { Box, Tab, Tabs } from "@mui/material"; +import DataFormatEditorTable from "./DataFormatEditorTable"; +import DataPreviewTable from "./DataPreviewTable"; +import TabPanel from "../../components/tabs/TabPanel"; +import "primereact/resources/themes/lara-light-cyan/theme.css"; +import "../../styles/tableStyles.css"; + +function DataPreviewAndFormatEditor({ + previewData, + setPreviewData, + availableColumnTypes, + columnMetadata, + setColumnMetadata, +}) { + const [columns, setColumns] = useState([]); + const [tabIndex, setTabIndex] = useState(0); + + useEffect(() => { + if (Array.isArray(previewData) && previewData.length > 0) { + const initialColumns = Object.keys(previewData[0]).map((key) => ({ + name: key, + })); + setColumns(initialColumns); + } + }, [previewData]); + + return ( + + setTabIndex(newValue)} + centered + > + + + + + + column.name)} + previewData={previewData} + /> + + + + + + + ); +} + +export default DataPreviewAndFormatEditor; diff --git a/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx b/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx deleted file mode 100644 index 32c40bd..0000000 --- a/frontend/src/pages/upload/DataPreviewAndSchemaEditor.jsx +++ /dev/null @@ -1,132 +0,0 @@ -import React, { useState, useEffect } from "react"; -import { - Box, - IconButton, - InputAdornment, - MenuItem, - Paper, - Select, - Table, - TableBody, - TableCell, - TableContainer, - TableHead, - TableRow, - TextField, - Tooltip, -} from "@mui/material"; -import EditIcon from "@mui/icons-material/Edit"; - -function DataPreviewAndSchemaEditor({ - previewData, - availableColumnTypes, - selectedColumnTypes, - onColumnsChange, -}) { - const [columns, setColumns] = useState([]); - - useEffect(() => { - if (Array.isArray(previewData) && previewData.length > 0) { - const initialColumns = Object.keys(previewData[0]).map((key) => ({ - name: key, - type: selectedColumnTypes[key] || "text", - isEditing: false, - })); - setColumns(initialColumns); - } - }, [previewData, selectedColumnTypes]); - - useEffect(() => { - onColumnsChange(columns); // Call the callback function when columns change - }, [columns]); - - const handleColumnTypeChange = (index, newType) => { - setColumns((prevColumns) => - prevColumns.map((column, colIndex) => - colIndex === index ? { ...column, type: newType } : column, - ), - ); - }; - - const handleEditClick = (index) => { - setColumns((prevColumns) => - prevColumns.map((column, colIndex) => - colIndex === index - ? { ...column, isEditing: !column.isEditing } - : column, - ), - ); - }; - - const handleColumnNameChange = (index, newName) => { - setColumns((prevColumns) => - prevColumns.map((column, colIndex) => - colIndex === index ? { ...column, name: newName } : column, - ), - ); - }; - - return ( - - - - - {columns.map((column, index) => ( - - - handleColumnNameChange(index, event.target.value) - } - variant="standard" - InputProps={{ - disableUnderline: true, - readOnly: !column.isEditing, - endAdornment: ( - - handleEditClick(index)}> - - - - ), - }} - style={{ cursor: "pointer" }} - /> - - - - - ))} - - - - {previewData.map((row, rowIndex) => ( - - {Object.values(row).map((value, cellIndex) => ( - {value} - ))} - - ))} - -
- - ); -} - -export default DataPreviewAndSchemaEditor; diff --git a/frontend/src/pages/upload/PreviewTable.jsx b/frontend/src/pages/upload/DataPreviewTable.jsx similarity index 81% rename from frontend/src/pages/upload/PreviewTable.jsx rename to frontend/src/pages/upload/DataPreviewTable.jsx index 7bf5c9b..7cdf2f8 100644 --- a/frontend/src/pages/upload/PreviewTable.jsx +++ b/frontend/src/pages/upload/DataPreviewTable.jsx @@ -5,16 +5,16 @@ import { Column } from "primereact/column"; import { InputTextarea } from "primereact/inputtextarea"; import "primereact/resources/themes/lara-light-cyan/theme.css"; import "../../styles/tableStyles.css"; -import { set } from "date-fns"; -function PreviewTable({ +function DataPreviewTable({ columnNames, previewData, - onChangePreviewData, + isEditCellMode, setIsEditingCell, + onEditCellData, }) { const columns = columnNames.map((name) => ({ - Header: name.toUpperCase(), + Header: name.replace(/_/g, " ").toUpperCase(), accessor: name, })); @@ -40,7 +40,7 @@ function PreviewTable({ let { rowData, newValue, field, originalEvent: event } = e; if (newValue.trim().length > 0) { - onChangePreviewData( + onEditCellData( data.findIndex((row) => row === rowData), field, newValue, @@ -60,25 +60,29 @@ function PreviewTable({ rowsPerPageOptions={[5, 10, 25, 50]} paginatorTemplate="RowsPerPageDropdown FirstPageLink PrevPageLink CurrentPageReport NextPageLink LastPageLink" currentPageReportTemplate="{first}-{last} of {totalRecords}" - editMode="cell" resizableColumns scrollable scrollHeight="flex" showGridlines tableStyle={{ minWidth: "150px" }} + {...(isEditCellMode ? { editMode: "cell" } : {})} > {columns.map((column) => ( cellEditor(options)} - onCellEditComplete={onCellEditComplete} bodyClassName="multi-line" + {...(isEditCellMode + ? { + editor: (options) => cellEditor(options), + onCellEditComplete: onCellEditComplete, + } + : {})} /> ))} ); } -export default PreviewTable; +export default DataPreviewTable; diff --git a/frontend/src/pages/upload/UploadPage.jsx b/frontend/src/pages/upload/UploadPage.jsx index 5bbd4dd..b5f6ce7 100644 --- a/frontend/src/pages/upload/UploadPage.jsx +++ b/frontend/src/pages/upload/UploadPage.jsx @@ -12,7 +12,7 @@ import CreateDataProfileWindow from "./CreateDataProfileWindow"; import DeleteDataProfileWindow from "./DeleteDataProfileWindow"; import DataProfileSelector from "./DataProfileSelector"; import FileUploader from "./FileUploader"; -import PreviewTable from "./PreviewTable"; +import DataPreviewTable from "./DataPreviewTable"; import { API_URL } from "../../utils/constants"; function UploadPage() { @@ -86,13 +86,13 @@ function UploadPage() { const handleCreateDataProfile = ( name, extractInstructions, - columnNamesAndTypes, + columnMetadata, ) => { axios .post(`${API_URL}data-profile/`, { name: name, extract_instructions: extractInstructions, - column_names_and_types: columnNamesAndTypes, + column_metadata: columnMetadata, }) .then((response) => { // Handle successful data profile creation @@ -222,11 +222,12 @@ function UploadPage() { {((columnNames && columnNames.length > 0) || previewData) && ( - )} diff --git a/frontend/src/styles/tableStyles.css b/frontend/src/styles/tableStyles.css index 93b476b..d03bea4 100644 --- a/frontend/src/styles/tableStyles.css +++ b/frontend/src/styles/tableStyles.css @@ -1,3 +1,7 @@ .multi-line { white-space: normal !important; } + +.p-dropdown-panel { + z-index: 1300 !important; +}