Merge pull request #73 from datajoint/external-storage

Add serialize/deserialize operations and bugfixes

.github/workflows/integration.yml

@@ -0,0 +1,38 @@
+name: Integration Tests
+on:
+  push:
+    branches:
+      - '**' # every branch
+      - '!stage*' # exclude branches beginning with stage
+  pull_request:
+    branches:
+      - '**' # every branch
+      - '!stage*' # exclude branches beginning with stage
+jobs:
+  CI:
+    if: github.event_name == 'push' || github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        matlab_version: ["R2019a"]
+        mysql_version: ["8.0.18", "5.7", "5.6"]
+        include:
+          - matlab_version: "R2018b"
+            mysql_version: "5.7"
+          - matlab_version: "R2016b"
+            mysql_version: "5.7"
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run primary tests
+        env:
+          MATLAB_UID: "1001"
+          MATLAB_GID: "116"
+          MATLAB_USER: ${{ secrets.matlab_user }}
+          MATLAB_HOSTID: ${{ secrets.matlab_hostid }}
+          MATLAB_VERSION: ${{ matrix.matlab_version }}
+          MYSQL_TAG: ${{ matrix.mysql_version }}
+          MATLAB_LICENSE: ${{ secrets[format('matlab_license_{0}', matrix.matlab_version)] }}
+          DOCKER_CLIENT_TIMEOUT: "120"
+          COMPOSE_HTTP_TIMEOUT: "120"
+        run: |
+          docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app

.gitignore

@@ -9,4 +9,4 @@ win.*
 macos*
 *.prj
 matlab.prf
-*.mltbx
+*.mltbx

LNX-docker-compose.yml

@@ -0,0 +1,105 @@
+# docker-compose -f LNX-docker-compose.yml --env-file LNX.env up --build --exit-code-from app
+version: '2.4'
+x-net: &net
+  networks:
+    - main
+services:
+  db:
+    <<: *net
+    image: datajoint/mysql:${MYSQL_TAG}
+    environment:
+      - MYSQL_ROOT_PASSWORD=simple
+  fakeservices.datajoint.io:
+    <<: *net
+    image: raphaelguzman/nginx:v0.0.10
+    environment:
+    - ADD_db_TYPE=DATABASE
+    - ADD_db_ENDPOINT=db:3306
+    depends_on:
+      db:
+        condition: service_healthy
+  app:
+    <<: *net
+    environment:
+    - MATLAB_LICENSE
+    - MATLAB_USER
+    - DJ_HOST=fakeservices.datajoint.io
+    - DJ_USER=root
+    - DJ_PASS=simple
+    - DJ_TEST_HOST=fakeservices.datajoint.io
+    - DJ_TEST_USER=datajoint
+    - DJ_TEST_PASSWORD=datajoint
+    image: raphaelguzman/matlab:${MATLAB_VERSION}-MIN
+    depends_on:
+      fakeservices.datajoint.io:
+        condition: service_healthy
+    user: ${MATLAB_UID}:${MATLAB_GID}
+    working_dir: /main
+    command:
+      - /bin/bash
+      - -c
+      - |
+        set -e
+        export ORIG_DIR=$$(pwd)
+        mkdir ~/Documents
+        cd /src
+        # Verify all mex have been updated
+        if [ "distribution/mexa64/mym.mexa64" -nt "distribution/mexmaci64/mym.mexmaci64" ]; then
+          echo "MACOS64 Mex binary appears outdated. Failing check..."
+          exit 1
+        fi
+        if [ "distribution/mexa64/mym.mexa64" -nt "distribution/mexw64/mym.mexw64" ]; then
+          echo "WIN64 Mex binary appears outdated. Failing check..."
+          exit 1
+        fi
+        # Compile mym, package into toolbox, and install
+        matlab -nodisplay -r "
+          try\
+            websave([tempdir 'GHToolbox.mltbx'],\
+                    ['https://github.com/datajoint/GHToolbox' \
+                     '/releases/download/' subsref(webread(['https://api.github.com/repos' \
+                                                            '/datajoint/GHToolbox' \
+                                                            '/releases/latest']),\
+                                                   substruct('.', 'tag_name')) \
+                     '/GHToolbox.mltbx']);\
+            matlab.addons.toolbox.installToolbox([tempdir 'GHToolbox.mltbx']);\
+            origDir = pwd;\
+            cd('distribution/mexa64');\
+            docs = help('mym');\
+            cd(origDir);\
+            ghtb.package('mym',\
+                         'Raphael Guzman',\
+                         '[email protected]',\
+                         ['MySQL API for MATLAB with support for BLOB objects'],\
+                         docs,\
+                         {'.vscode', '.git', '.gitignore', 'build', 'lib', 'maria-plugin',\
+                          'mex_compilation', 'mysql-connector', 'notebook', 'src', 'zlib',\
+                          '*.txt', '*.env', '*.prf', '*.md', '*.yml', 'tests', '.github',\
+                          'distribution/mexa64/libmysqlclient.so.18.4.'},\
+                         @() strjoin(arrayfun(@(x) num2str(x),\
+                                              cell2mat(struct2cell(mym('version'))),\
+                                              'uni', false),\
+                                     '.'),\
+                         {'distribution/mexa64', 'distribution/mexmaci64',\
+                          'distribution/mexw64', 'mym.m'},\
+                         'toolboxVersionDir', 'distribution/mexa64',\
+                         'toolboxRootDir', '.');\
+            matlab.addons.toolbox.installToolbox('mym.mltbx');\
+            cd(getenv('ORIG_DIR'));\
+            addpath('tests');\
+            res=run(Main);\
+            disp(res);\
+            if all([res.Passed])  exit, else exit(1), end;\
+            ,\
+          catch ME,\
+            disp(getReport(ME, 'extended'));\
+            exit(1);\
+            ,\
+          end;\
+        "
+    mac_address: $MATLAB_HOSTID
+    volumes:
+      - ./tests:/main/tests
+      - .:/src
+networks:
+  main:

README.md

@@ -12,6 +12,47 @@ work of Robert Almgren from University of Toronto [sourceforge project](http://s
 
 See mym.m for further documentation.
 
+## Running Tests Locally
+
+* Create an `.env` with desired development environment values e.g.
+``` sh
+MATLAB_USER=raphael
+MATLAB_LICENSE="#\ BEGIN----...---------END" # For image usage instructions see https://github.com/guzman-raphael/matlab, https://hub.docker.com/r/raphaelguzman/matlab
+MATLAB_VERSION=R2018b
+MATLAB_HOSTID=XX:XX:XX:XX:XX:XX
+MATLAB_UID=1000
+MATLAB_GID=1000
+MYSQL_TAG=5.7
+```
+* `cp local-docker-compose.yml docker-compose.yml`
+* `docker-compose up` (Note configured `JUPYTER_PASSWORD`)
+* Select a means of running MATLAB e.g. Jupyter Notebook, GUI, or Terminal (see bottom)
+* Run desired tests. Some examples are as follows:
+
+| Use Case                     | MATLAB Code                                                                    |
+| ---------------------------- | ------------------------------------------------------------------------------ |
+| Run all tests                | `run(Main)`                                                              |
+| Run one class of tests       | `run(TestTls)`                                                           |
+| Run one specific test        | `runtests('TestTls/TestTls_testInsecureConn')`                                   |
+| Run tests based on test name | `import matlab.unittest.TestSuite;`<br>`import matlab.unittest.selectors.HasName;`<br>`import matlab.unittest.constraints.ContainsSubstring;`<br>`suite = TestSuite.fromClass(?Main, ... `<br><code>&nbsp;&nbsp;&nbsp;&nbsp;</code>`HasName(ContainsSubstring('Conn')));`<br>`run(suite)`|
+
+
+### Launch Jupyter Notebook
+* Navigate to `localhost:8888`
+* Input Jupyter password
+* Launch a notebook i.e. `New > MATLAB`
+
+
+### Launch MATLAB GUI (supports remote interactive debugger)
+* Shell into `mym_app_1` i.e. `docker exec -it mym_app_1 bash`
+* Launch Matlab by runnning command `matlab`
+
+
+### Launch MATLAB Terminal
+* Shell into `mym_app_1` i.e. `docker exec -it mym_app_1 bash`
+* Launch Matlab with no GUI by runnning command `matlab -nodisplay`
+
+
 ## Installation
 
 ### (Recommended) Using GHToolbox (FileExchange Community Toolbox)

distribution/mexa64/mym.m

@@ -43,8 +43,13 @@
 %     host:     default is local host. Use colon for port number
 %     user:     default is Unix login name.
 %     password: default says connect without password.
-%   Examples: mym('open','arkiv')     %  connect on default port
-%             mym('open','arkiv:2215')
+%     use_tls:  (optional) TLS encryption type. Values are as follows:
+%                - 'true':           TLS encryption is required.
+%                - 'false':          TLS encryption is disabled.
+%                - 'nan'(default):   TLS encryption utilized if available.
+%   Examples: mym('open','arkiv')      %  connect on default port
+%             mym('open','arkiv:2215') %  TLS Preferred
+%             mym('open','arkiv','root','simple','true') %  TLS Required  
 %   If successful, open returns 0 if successful, and throw an error
 %   otherwise. The program can maintain up to 20 independent connections.
 %   Any command may be preceded by a connection handle -- an integer from 0
@@ -67,7 +72,8 @@
 %   Example:  mym('use cme')
 % mym('status')
 % -------------
-%   Display information about the connection and the server.
+%   Display information about the connection and the server. Connections
+%   utilizing TLS encryption will be displayed as '(encrypted)'.
 %   Return  0  if connection is open and functioning
 %           1  if connection is closed
 %           2  if should be open but we cannot ping the server
@@ -119,6 +125,14 @@
 %            solution is to use the following command:
 %             mym(INSERT INTO tbl(id,txt) VALUES(1000,"{S}")','abc{dfg}h');
 %
+% mym('serialize {placeholder1}, ...',matlab_variable1, ...)
+% -------------
+%   Return : cell array of vectors of type ubyte, a vector per matlab variable 
+%
+% mym('deserialize', serialized_input)
+% -------------
+%   Return : matlab variable of the appropriate type 
+%
 % mym('version')
 % --------------
 % Displays the version of mym when no output arguments are given.
@@ -154,7 +168,7 @@
 %         - use std::max<int>(a, b) instead of max(a, b)
 % v1.0.5  - added the preamble 'u', permitting to save binary fields without using compression
 %         - corrected a bug in mym('closeall')
-%         - corrected various mistakes in the help file (thanks to J�rg Buchholz)
+%         - corrected various mistakes in the help file (thanks to Jörg Buchholz)
 % v1.0.4  corrected the behaviour of mYm with time fields, now return a string dump of the field
 % v1.0.3 	minor corrections
 % v1.0.2  put mYm under GPL license, official release

distribution/mexmaci64/mym.m

@@ -43,8 +43,13 @@
 %     host:     default is local host. Use colon for port number
 %     user:     default is Unix login name.
 %     password: default says connect without password.
-%   Examples: mym('open','arkiv')     %  connect on default port
-%             mym('open','arkiv:2215')
+%     use_tls:  (optional) TLS encryption type. Values are as follows:
+%                - 'true':           TLS encryption is required.
+%                - 'false':          TLS encryption is disabled.
+%                - 'nan'(default):   TLS encryption utilized if available.
+%   Examples: mym('open','arkiv')      %  connect on default port
+%             mym('open','arkiv:2215') %  TLS Preferred
+%             mym('open','arkiv','root','simple','true') %  TLS Required  
 %   If successful, open returns 0 if successful, and throw an error
 %   otherwise. The program can maintain up to 20 independent connections.
 %   Any command may be preceded by a connection handle -- an integer from 0
@@ -67,7 +72,8 @@
 %   Example:  mym('use cme')
 % mym('status')
 % -------------
-%   Display information about the connection and the server.
+%   Display information about the connection and the server. Connections
+%   utilizing TLS encryption will be displayed as '(encrypted)'.
 %   Return  0  if connection is open and functioning
 %           1  if connection is closed
 %           2  if should be open but we cannot ping the server
@@ -119,6 +125,14 @@
 %            solution is to use the following command:
 %             mym(INSERT INTO tbl(id,txt) VALUES(1000,"{S}")','abc{dfg}h');
 %
+% mym('serialize {placeholder1}, ...',matlab_variable1, ...)
+% -------------
+%   Return : cell array of vectors of type ubyte, a vector per matlab variable 
+%
+% mym('deserialize', serialized_input)
+% -------------
+%   Return : matlab variable of the appropriate type 
+%
 % mym('version')
 % --------------
 % Displays the version of mym when no output arguments are given.
@@ -154,7 +168,7 @@
 %         - use std::max<int>(a, b) instead of max(a, b)
 % v1.0.5  - added the preamble 'u', permitting to save binary fields without using compression
 %         - corrected a bug in mym('closeall')
-%         - corrected various mistakes in the help file (thanks to J�rg Buchholz)
+%         - corrected various mistakes in the help file (thanks to Jörg Buchholz)
 % v1.0.4  corrected the behaviour of mYm with time fields, now return a string dump of the field
 % v1.0.3 	minor corrections
 % v1.0.2  put mYm under GPL license, official release