Merge pull request #24 from apdn7/feature/4.7.3

Feature/4.7.3

README.md

@@ -5,7 +5,7 @@
 # Analysis Platform
 
 ```
-2024-07-22: version 4.7.1
+2024-09-18: version 4.7.3
 ```
 
 Analysis Platform is an open source web application to import, connect and visualize factory IoT data. It helps to collect, link and integrate data from multiple data sources.
@@ -65,21 +65,21 @@ Windows embeddable package, running the batch file is regarded as you agreed to
 
 ## Requirements
 
-* Windows (Tested with Windows 10)
+* Windows (Tested with Windows 10), Windoes Server
 * Python (>=3.6) (Tested with Python 3.9.0, pip 22.2)
   * If you are going to use the Windows embeddable package, you do not have to install Python in your machine.  
-  (See section: For users: Run the Analysis Platform with the Windows embeddable package)
+  (See section: For users)
 
 ## Getting Started
 
 You can install AP+DN7 on your PC using installer (batch file)
 
-* [APDN7_installer.zip](https://github.com/user-attachments/files/15897388/APDN7_installer.zip)
+* [APDN7_installer.zip](https://github.com/user-attachments/files/17042671/APDN7_installer.zip)
 
 For detailed instruction, please refer below documents (PDF):
 
-* EN: [Getting Started](https://github.com/apdn7/AnalysisPlatform/releases/download/v4.7.0/1_AP+DN7_GettingStarted_OSS_470_en.pdf) / [Data Import & Link Settings](https://github.com/apdn7/AnalysisPlatform/files/15073708/AP%2BDN7_Setting_Manual_461_En.pdf) / [FPP](https://github.com/apdn7/AnalysisPlatform/files/9806413/AP%2BDN7_User_Manual_FPP_En.pdf) / [FAQ](https://github.com/apdn7/AnalysisPlatform/files/9314734/AP%2BDN7_FAQ_EN_v1.0.pdf) / [Quick reference](https://github.com/apdn7/AnalysisPlatform/files/9314737/AP%2BDN7_v4.0_QuickReference_EN.pdf)
-* JA: [Getting Started](https://github.com/apdn7/AnalysisPlatform/releases/download/v4.7.0/1_AP+DN7_GettingStarted_OSS_470_Ja.pdf) / [Data Import & Link Settings](https://github.com/apdn7/AnalysisPlatform/files/15074076/AP%2BDN7_Setting_Manual_461_jp.pdf) / [FPP](https://github.com/apdn7/AnalysisPlatform/files/9806414/AP-DN7_FPP_Jp0.pdf) / [FAQ](https://github.com/apdn7/AnalysisPlatform/files/9256487/AP%2BDN7_FAQ_JP_v1.0.pdf) / [Quick reference](https://github.com/apdn7/AnalysisPlatform/files/9314735/AP%2BDN7_v4.0_QuickReference_JP.pdf)
+* EN: [Getting Started](https://github.com/user-attachments/files/17042441/1_AP%2BDN7_GettingStarted_OSS_473_en.pdf) / [Data Import & Link Settings](https://github.com/apdn7/AnalysisPlatform/files/15073708/AP%2BDN7_Setting_Manual_461_En.pdf) / [FPP](https://github.com/apdn7/AnalysisPlatform/files/9806413/AP%2BDN7_User_Manual_FPP_En.pdf) / [FAQ](https://github.com/apdn7/AnalysisPlatform/files/9314734/AP%2BDN7_FAQ_EN_v1.0.pdf) / [Quick reference](https://github.com/apdn7/AnalysisPlatform/files/9314737/AP%2BDN7_v4.0_QuickReference_EN.pdf)
+* JA: [Getting Started](https://github.com/user-attachments/files/17042443/1_AP%2BDN7_GettingStarted_OSS_473_ja.pdf) / [Data Import & Link Settings](https://github.com/apdn7/AnalysisPlatform/files/15074076/AP%2BDN7_Setting_Manual_461_jp.pdf) / [FPP](https://github.com/apdn7/AnalysisPlatform/files/9806414/AP-DN7_FPP_Jp0.pdf) / [FAQ](https://github.com/apdn7/AnalysisPlatform/files/9256487/AP%2BDN7_FAQ_JP_v1.0.pdf) / [Quick reference](https://github.com/apdn7/AnalysisPlatform/files/9314735/AP%2BDN7_v4.0_QuickReference_JP.pdf)
 
 ### For users
 

RELEASE.md

@@ -1,5 +1,96 @@
 # Releases
 
+## v4.7.3
+
+New features and Improvements
+
+* (Common)
+  * On-demand filter can now be applied when using `Jump` Function  
+  <img src="https://github.com/user-attachments/assets/4e441d6d-3d30-4b0b-b658-0d0dd8afac67" alt="JumpODF" width="600">
+  * Visualization pages now opens with no bookmark loaded by default
+  * Improved auto-scrolling after showing graph and scrolling buttons
+  * When copying settings between pages, the time Window Length and Interval are now adjusted to keep the time period the same
+  * Datetime can now be selected on Display Config page
+  * Datetime can now be calculated as CT for PCP, SkD, PCA, GL pages
+  * Copying User setting now copies the Outlier cleansing settings as well
+  * Copying User setting from Stratified Plot to Ridgeline Plot now keeps the target period settings
+  * X and Y axis can now be changed in ScatterPlot and HeatMap pages
+* (FPP)
+  * Parallel Coordinate Plot now uses Auto Range by default
+  * All columns in the dataframe can now be selected for X-axis ordering
+  * The hover popup no longer go out of the browser's viewport
+* (PCP)
+  * Enabled Y-axis scaling (Full range, Graph config, etc)
+  * The latest 20 srecords within the selected data are now displayed in a table  
+  <img src="https://github.com/user-attachments/assets/f442d950-0da4-46d3-90ca-c6c536c9a983" alt="PCPDataPreview" width="600">
+* (SkD)
+  * The hover message in SkD now shows the positive/negative relationship between variables
+* (AgP)
+  * AgP is now able to use a common scale for histogram and line charts respectively
+* (HMp)
+  * X and Y values are now shown on hovers in HeatMap page
+* (Config)
+  * New data types can now be used in Process Configuration: `Judge` and `Boolean`
+  * `Merge mode` can now be used for databases
+  * Sample Data in `Function` configuration can now be copied into a spreadsheet
+  * Backup-Restore: Imported data can now be backed up and restored within a specified time period
+  * Process Configuration for registered processes can now be viewed without connecting to the datasource
+  * Process Configuration can now be downloaded instead of copying to clipboard when APDN7 is accessed through an IP address  
+  <img src="https://github.com/user-attachments/assets/b1c2fdf8-24f8-41c5-99be-eddceff8b9ec" alt="downloadconfig" width="600">
+  * Sample data in Function Configuration page now shows the top 50 results by unique X-Y pairs
+  * Merge function now works better with mixed datatypes between X and Y
+  * Process Configuration modal now has explanation for some data types
+  * Register By File now jumps to FPP instead of CHM after importing is finished if dummy_datetime is present
+  * Optimized the speed for data link using datetime-cutoff
+  * Improved responsive styling of the Function Configuration modal
+  * Filter name in Filter Configuration page is now auto-filled when selecting a column
+  * Master filters are not automatically selected as filters and labels in Show graph pages. PartName, PartNo, LineName, LineNo are selected for the first process only.
+  * Datasource preview now shows rows with NA values
+  * In Function Configuration modal, function sample data rows are not sorted in X -> Y -> Result priority, and NA values are at the bottom
+  * Function Configuration now shows both local and system names for columns in the input selection
+* (Misc)
+  * Now `Register by File` page supports V2 CSV files
+  * Support new data source: Software workshop databases
+  * The Jump Function modal can now be filtered, and has Auto Select, Select all checkboxes
+
+Bug fixes
+
+* (Common)
+  * Fixed an issue where a column is not moved to the top of the list when selected as Facet or Div in Show graph pages
+* (FPP)
+  * Fixed an issue where outlier points do not lie at the edge of FPP charts
+* (PCP)
+  * Fixed a bug where categories are wrongly sorted
+  * Fixed an issue where the "Move next to target variable" button in Parallel Coordinate Plot fails to work
+  * Fixed a display bug where NA and Infinite values have the wrong color in Parallel Coordinate Plot
+* (SkD)
+  * Fixed an issue where SankeyDiagram fails to show binary classification mode when there are NA values in the data
+* (RLP)
+  * Fixed an issue where the Graph settings option does not work correctly for RLP
+* (Config)
+  * Data sources and processes created through Register By File page can now be previewed
+  * Autolink fails due to lack of delta_time and cut_off fields
+  * Fixed a bug where Auto Select does not work properly in Process Config modal
+  * Fixed a bug where Merge Mode returns an error when merging Float data into an Integer column
+  * Fixed an issue where datetime format is not applied after changing the datatype of a column in Process Config modal
+  * Fixed an issue where job status is not shown in Process Config page
+  * Fixed a bug where confirmation modal appears even when no changes are made to a function
+  * Fixed a display bug where long column names cause text to overflow in Function Config
+  * Fixed an issue where a failed validation remains even after changing function
+  * Fixed an issue where X and Y cannot be selected after selecting me.type.convert(t)
+  * Fixed a bug where filling a process name in Process Config screen does not fill the process name field in the Process Config Modal
+  * Fixed an issue where duplicate column names are not automatically parsed in Process Config page and data import fails against duplicate column names with "_" character
+  * Fixed a problem where preview of database leads to an error if database has boolean data
+* (HMp)
+  * Fixed an issue in HeatMap page where a wrong message is shown when showing graph
+  * Fixed an issue where HeatMap page shows a wrong color scale for binary and integer cases
+* (Misc)
+  * Fixed a bug where using Jump Function alters the copied setting
+  * Fixed an issue where child processes appear in Table Viewer and Filter pages
+  * Fixed a display bug where milliseconds are displayed for time data in HeatMap and ScatterPlot pages
+* (ScP)
+  * Fixed an issue where ScatterPlot shows a wrong color scale when Integer is selected as Color
+
 ## v4.7.1
 
 Improvements

VERSION

@@ -1,5 +1,5 @@
-v4.7.1.234.801d8447
-1
-OSS
+v4.7.3.239.3fb04467
+1
+OSS
 
 

_original_path_list.log

@@ -6,6 +6,7 @@
 .\format.bat
 .\LICENSE.md
 .\main.py
+.\package-lock.json
 .\package.json
 .\pyper.py
 .\pyproject.toml
@@ -82,11 +83,13 @@
 .\ap\api\setting_module\services\equations.py
 .\ap\api\setting_module\services\factory_import.py
 .\ap\api\setting_module\services\filter_settings.py
+.\ap\api\setting_module\services\master_data_transform_pattern.py
 .\ap\api\setting_module\services\polling_frequency.py
 .\ap\api\setting_module\services\process_delete.py
 .\ap\api\setting_module\services\save_load_user_setting.py
 .\ap\api\setting_module\services\show_latest_record.py
 .\ap\api\setting_module\services\shutdown_app.py
+.\ap\api\setting_module\services\software_workshop_etl_services.py
 .\ap\api\setting_module\services\v2_etl_services.py
 .\ap\api\setting_module\services\__init__.py
 .\ap\api\table_viewer\controllers.py
@@ -113,6 +116,7 @@
 .\ap\common\common_utils.py
 .\ap\common\constants.py
 .\ap\common\cryptography_utils.py
+.\ap\common\datetime_format_utils.py
 .\ap\common\disk_usage.py
 .\ap\common\logger.py
 .\ap\common\memoize.py
@@ -211,12 +215,14 @@
 .\ap\script\hide_exe_root_folder.py
 .\ap\script\migrate_cfg_data_source_csv.py
 .\ap\script\migrate_cfg_process.py
+.\ap\script\migrate_cfg_process_column.py
 .\ap\script\migrate_csv_datatype.py
 .\ap\script\migrate_csv_dummy_datetime.py
 .\ap\script\migrate_csv_save_graph_settings.py
 .\ap\script\migrate_delta_time.py
 .\ap\script\migrate_m_function.py
 .\ap\script\migrate_process_file_name_column.py
+.\ap\script\setup_for_e2e.py
 .\ap\script\dummy_data\dummy_sql.py
 .\ap\script\hot_fix\fix_db_issues.py
 .\ap\script\r_scripts\analytics.r
@@ -237,6 +243,12 @@
 .\ap\setting_module\services\process_config.py
 .\ap\setting_module\services\register_from_file.py
 .\ap\setting_module\services\trace_config.py
+.\ap\setting_module\services\backup_and_restore\backup.py
+.\ap\setting_module\services\backup_and_restore\backup_file_manager.py
+.\ap\setting_module\services\backup_and_restore\duplicated_check.py
+.\ap\setting_module\services\backup_and_restore\jobs.py
+.\ap\setting_module\services\backup_and_restore\restore.py
+.\ap\setting_module\services\backup_and_restore\__init__.py
 .\ap\static\aggregate_plot\css\aggregate_plot.css
 .\ap\static\aggregate_plot\js\aggregate_plot.js
 .\ap\static\aggregate_plot\js\aggregation_chart.js
@@ -286,6 +298,7 @@
 .\ap\static\common\css\jsuites.css
 .\ap\static\common\css\jump_function.css
 .\ap\static\common\css\main.css
+.\ap\static\common\css\multi_level_dropdown.css
 .\ap\static\common\css\pagination.css
 .\ap\static\common\css\select2.min.css
 .\ap\static\common\css\shepherd.css
@@ -560,7 +573,9 @@
 .\ap\static\scatter_plot\js\scp_heatmap_plot.js
 .\ap\static\scatter_plot\js\scp_violin_plot.js
 .\ap\static\setting_module\css\background_process.css
+.\ap\static\setting_module\css\backup_restore_modal.css
 .\ap\static\setting_module\css\config_view.css
+.\ap\static\setting_module\css\data_type_dropdown.css
 .\ap\static\setting_module\css\filter_visualization.css
 .\ap\static\setting_module\css\index.css
 .\ap\static\setting_module\css\master_cfg.css
@@ -597,11 +612,21 @@
 .\ap\static\setting_module\js\graph_visualization.js
 .\ap\static\setting_module\js\master_config.js
 .\ap\static\setting_module\js\npm.js
+.\ap\static\setting_module\js\parse_data_utils.js
 .\ap\static\setting_module\js\predict_data_type.js
+.\ap\static\setting_module\js\process_config_section.js
 .\ap\static\setting_module\js\proc_config.js
+.\ap\static\setting_module\js\proc_config_date_time_format.js
 .\ap\static\setting_module\js\proc_config_modals.js
 .\ap\static\setting_module\js\register_from_file.js
+.\ap\static\setting_module\js\system.js
 .\ap\static\setting_module\js\trace_config.js
+.\ap\static\setting_module\js\data_type_dropdown\constant.js
+.\ap\static\setting_module\js\data_type_dropdown\controller.js
+.\ap\static\setting_module\js\data_type_dropdown\core.js
+.\ap\static\setting_module\js\data_type_dropdown\event.js
+.\ap\static\setting_module\js\data_type_dropdown\helper.js
+.\ap\static\setting_module\js\data_type_dropdown\type_definition.js
 .\ap\static\table_viewer\css\table_viewer.css
 .\ap\static\table_viewer\js\table_viewer.js
 .\ap\static\table_viewer\lang\English.json
@@ -660,6 +685,7 @@
 .\ap\templates\setting_module\proc_config.html
 .\ap\templates\setting_module\proc_config_modals.html
 .\ap\templates\setting_module\register_by_file.html
+.\ap\templates\setting_module\system.html
 .\ap\templates\setting_module\terms_of_use.html
 .\ap\templates\setting_module\trace_config.html
 .\ap\templates\setting_module\_filter_line.html
@@ -782,13 +808,14 @@
 .\ap\translations\zh_Hans_CN\LC_MESSAGES\messages.po
 .\ap\translations\zh_Hant_TW\LC_MESSAGES\messages.mo
 .\ap\translations\zh_Hant_TW\LC_MESSAGES\messages.po
-.\bridge\services\sql\sql_generator.py
-.\bridge\services\sql\transaction_query_builder.py
 .\data_files\19.m_function.tsv
 .\init\app.sqlite3
 .\init\sqlite3.dll
 .\lang\message.pot
+.\migrations\versions\8600fce518a6_add_datetime_format_column_into_cfg_.py
+.\migrations\versions\fd863d0b8735_add_is_file_path_column_into_cfg_data_.py
 .\requirements\common.txt
+.\requirements\dev.txt
 .\requirements\oss_dev.txt
 .\requirements\oss_prod.txt
 .\sample_data\AgP_sample_data\AgP_sample.tsv

ap/__init__.py

@@ -188,6 +188,14 @@ def init_engine(app, uri, **kwargs):
     def do_begin(dbapi_conn):
         dbapi_conn.execute('BEGIN IMMEDIATE')
 
+    @event.listens_for(db_engine, 'commit')
+    def do_expire(dbapi_conn):
+        """
+        Expire all objects in `db.session` everytime meta session perform a commit.
+        This makes `db.session` removes all cached and queries to database again to get the newest objects
+        """
+        db.session.expire_all()
+
     return db_engine
 
 
@@ -239,7 +247,8 @@ def create_app(object_name=None, is_main=False):
     from .sankey_plot import create_module as sankey_create_module
     from .scatter_plot import create_module as scatter_plot_create_module
     from .script.migrate_cfg_data_source_csv import migrate_cfg_data_source_csv
-    from .script.migrate_cfg_process import migrate_cfg_process_add_is_show_file_name
+    from .script.migrate_cfg_process import migrate_cfg_process
+    from .script.migrate_cfg_process_column import migrate_cfg_process_column
     from .script.migrate_csv_datatype import migrate_csv_datatype
     from .script.migrate_csv_dummy_datetime import migrate_csv_dummy_datetime
     from .script.migrate_csv_save_graph_settings import migrate_csv_save_graph_settings
@@ -386,7 +395,8 @@ def get_locale():
         migrate_cfg_process_column_add_column_raw_name(app.config[APP_DB_FILE])
         migrate_cfg_process_column_add_column_type(app.config[APP_DB_FILE])
         migrate_cfg_process_column_add_parent_id(app.config[APP_DB_FILE])
-        migrate_cfg_process_add_is_show_file_name(app.config[APP_DB_FILE])
+        migrate_cfg_process_column(app.config[APP_DB_FILE])
+        migrate_cfg_process(app.config[APP_DB_FILE])
 
         # migrate function data
         migrate_m_function_data(app.config[APP_DB_FILE])

ap/api/aggregate_plot/controllers.py

@@ -10,6 +10,7 @@
 from ap.api.categorical_plot.services import customize_dict_param
 from ap.api.common.services.show_graph_database import get_config_data
 from ap.api.common.services.show_graph_jump_function import get_jump_emd_data
+from ap.api.common.services.show_graph_services import judge_data_conversion
 from ap.api.trace_data.services.csv_export import to_csv
 from ap.common.constants import (
     ARRAY_FORMVAL,
@@ -122,6 +123,8 @@ def data_export(export_type):
             None,
             max_graph_config[MaxGraphNumber.AGP_MAX_GRAPH.name],
         )
+        # export original value of judge variable
+        agp_df = judge_data_conversion(agp_df, graph_param, revert=True)
         end_proc_id = int(agp_dat[ARRAY_FORMVAL][0][END_PROC])
         proc_name = graph_param.dic_proc_cfgs[end_proc_id].shown_name
         csv_list_name.append('{}.{}'.format(proc_name, export_type))

ap/api/aggregate_plot/services.py

@@ -21,6 +21,7 @@
     filter_cat_dict_common,
     get_data_from_db,
     get_filter_on_demand_data,
+    judge_data_conversion,
 )
 from ap.api.scatter_plot.services import gen_df
 from ap.common.constants import (
@@ -149,14 +150,19 @@ def gen_agp_data(root_graph_param: DicParam, dic_param, df=None, max_graph=None)
     dic_param[ARRAY_PLOTDATA] = dic_data
     dic_param[IS_GRAPH_LIMITED] = is_graph_limited
     # calc y scale
-    min_max_list, all_graph_min, all_graph_max = calc_raw_common_scale_y(dic_param[ARRAY_PLOTDATA], str_cols)
+    min_max_list, all_graph_min, all_graph_max, max_common_y_scale_count = calc_raw_common_scale_y(
+        dic_param[ARRAY_PLOTDATA],
+        str_cols,
+        is_get_common_y_scale_count=True,
+    )
     calc_scale_info(
         graph_param.dic_proc_cfgs,
         dic_param[ARRAY_PLOTDATA],
         min_max_list,
         all_graph_min,
         all_graph_max,
         str_cols,
+        max_common_y_scale_count=max_common_y_scale_count,
     )
 
     dic_param = get_filter_on_demand_data(dic_param)
@@ -199,6 +205,7 @@ def gen_df_direct_term(root_graph_param, dic_param, dic_cat_filters, use_expired
             dic_cat_filters,
             _use_expired_cache=use_expired_cache,
         )
+        df_term = judge_data_conversion(df_term, root_graph_param)
 
         df_term[DIVIDE_FMT_COL] = f'{term[START_DT]} | {term[END_DT]}'
 

ap/api/analyze/services/pca.py

@@ -4,6 +4,7 @@
 from sklearn.preprocessing import StandardScaler
 
 from ap.api.common.services.show_graph_services import (
+    convert_datetime_to_ct,
     customize_dic_param_for_reuse_cache,
     filter_cat_dict_common,
     get_data_from_db,
@@ -218,6 +219,7 @@ def gen_trace_data(graph_param, orig_graph_param, dic_cat_filters, use_expired_c
 
     # get data from database
     df, actual_record_number, unique_serial = get_trace_data(graph_param, dic_cat_filters, use_expired_cache)
+    convert_datetime_to_ct(df, graph_param)
 
     dic_var_name = {}
     for col_alias, id in ids.items():