change shema, integrate data profiling

backend/alembic/versions/94af7f09e896_configure_data_profile_model.py

@@ -0,0 +1,30 @@
+"""configure data profile model
+
+Revision ID: 94af7f09e896
+Revises: a7364adb18ea
+Create Date: 2024-01-20 22:03:53.600966
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "94af7f09e896"
+down_revision = "a7364adb18ea"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.alter_column(
+        "data_profiles", "description", new_column_name="extract_instructions"
+    )
+    op.add_column("data_profiles", sa.Column("table_id", sa.Integer))
+
+
+def downgrade():
+    op.alter_column(
+        "data_profiles", "extract_instructions", new_column_name="description"
+    )
+    op.drop_column("data_profiles", "table_id")

backend/database/data_profile_manager.py

@@ -34,7 +34,7 @@ def create_dataprofile(self, data_profile_data: DataProfileCreateRequest):
             name=data_profile_data.name,
             file_type=data_profile_data.file_type,
             organization_id=data_profile_data.organization_id,
-            description=data_profile_data.description,
+            extract_instructions=data_profile_data.extract_instructions,
         )
         self.session.add(new_data_profile)
         self.session.commit()

backend/models/data_profile.py

@@ -14,7 +14,7 @@ class DataProfile(Base):
     - name: The name of the data profile.
     - file_type: The type of file associated with the data profile.
     - organization_id: The organization associated with the data profile.
-    - description: The description of the data profile.
+    - extract_instructions: The instructions for extracting data from the file.
     The class also is converting the model instance into a dictionary.
     """
 
@@ -23,7 +23,8 @@ class DataProfile(Base):
     name = Column(String, unique=True)
     file_type = Column(String)
     organization_id = Column(Integer, ForeignKey("organizations.id"))
-    description = Column(String)  # New description column
+    extract_instructions = Column(String)
+    table_id = Column(Integer)
 
     __table_args__ = (
         UniqueConstraint("name", "organization_id", name="uq_name_organization_id"),
@@ -38,15 +39,16 @@ def to_dict(self):
             "name": self.name,
             "file_type": self.file_type,
             "organization_id": self.organization_id,
-            "description": self.description,
+            "extract_instructions": self.extract_instructions,
+            "table_id": self.table_id,
         }
 
 
 class DataProfileCreateRequest(BaseModel):
     name: str
-    description: str
+    extract_instructions: str
 
 
 class DataProfileCreateResponse(BaseModel):
     name: str
-    description: str
+    extract_instructions: str

backend/routes/data_profile_routes.py

@@ -40,7 +40,7 @@ async def get_data_profiles_by_org_id(current_user: User = Depends(get_current_u
 
 
 @data_profile_router.post("/data-profile/")
-async def save_data_profiles(
+async def save_data_profile(
     request: DataProfileCreateRequest, current_user: User = Depends(get_current_user)
 ) -> DataProfileCreateResponse:
     with DatabaseManager() as session:
@@ -52,15 +52,15 @@ async def save_data_profiles(
 
         new_data_profile = DataProfile(
             name=request.name,
-            description=request.description,
+            extract_instructions=request.extract_instructions,
             organization_id=current_user.organization_id,
         )
         created_data_profile = data_profile_manager.create_dataprofile(new_data_profile)
 
         # Make sure to pass the fields as keyword arguments
         response = DataProfileCreateResponse(
             name=created_data_profile.name,
-            description=created_data_profile.description,
+            extract_instructions=created_data_profile.extract_instructions,
         )
         return response
 

frontend/src/pages/upload/CreateDataProfilePage.jsx

@@ -34,7 +34,7 @@ function CreateDataProfilePage({ open, onClose, onCreate }) {
       sampleFiles.forEach((file) => {
         formData.append("files", file); // Append each file
       });
-      formData.append("instructions", extractInstructions);
+      formData.append("extract_instructions", extractInstructions);
 
       axios
         .post(`${API_URL}data-profiles/preview/`, formData, {

frontend/src/pages/upload/UploadPage.jsx

@@ -41,7 +41,7 @@ function UploadPage() {
     axios
       .post(`${API_URL}data-profile/`, {
         name: name,
-        description: extractInstructions,
+        extract_instructions: extractInstructions,
       })
       .then((response) => {
         // Handle successful data profile creation