(TA-Lib#43) New test-dist.py script to verify release candidate

scripts/install_tests/__init__.py

@@ -0,0 +1 @@
+# install_tests/__init__.py

scripts/install_tests/python.py

@@ -0,0 +1,73 @@
+import os
+
+def log_params(host: str, package_file_path: str, temp_dir: str, version: str, sudo_pwd: str):
+    print(f"Testing ta-lib-python installation on {host}")
+    # Never display or log sudo_pwd, but want to know if it was specified.
+    if sudo_pwd:
+        hidden_sudo_pwd = "(hidden)"
+    else:
+        hidden_sudo_pwd = "\"\""
+
+    print(f"  package_file_path={package_file_path}")
+    print(f"  temp_dir={temp_dir}")
+    print(f"  version={version}")
+    print(f"  sudo_pwd={hidden_sudo_pwd}")
+
+    # Create a dummy file "PARAMS" into temp_dir to help debugging.
+    with open(os.path.join(temp_dir, "PARAMS"), "w") as f:
+        f.write(f"package_file_path={package_file_path}\n")
+        f.write(f"temp_dir={temp_dir}\n")
+        f.write(f"version={version}\n")
+        f.write(f"sudo_pwd={hidden_sudo_pwd}\n")
+
+    return
+
+def test_python_windows(package_file_path: str, temp_dir: str, version: str, sudo_pwd: str):
+    # Test installation procedure for ta-lib-python and validate
+    # that this ta-lib package release candidate is OK.
+    #
+    # Parameters
+    # ----------
+    # package_file_path is the ta-lib-{version}-win64.zip
+    #
+    # temp_dir is an empty directory that can be used
+    # for most intermediate file operations.
+    #
+    # version is the "0.0.0" string expected to be returned
+    # by the installed ta-lib package.
+    #
+    # sudo_pwd can be pipeline into a "sudo -S" command.
+    # If sudo_pwd is an empty string, then call "sudo" without
+    # "-S" and let it be interactive.
+    # More info:
+    #    https://stackoverflow.com/questions/60807449/run-github-action-as-sudo
+    #
+    # Test Behavior
+    # -------------
+    # Must re-install/upgrade if ta-lib is already installed.
+    #
+    # Recommended to create a venv in temp_dir to simulate
+    # a user setup.
+    #
+    # Leave all intermediate files in temp_dir after the test
+    # (might be useful for debugging).
+    #
+    # Just sys.exit(1) on any problem discovered.
+
+    log_params("Windows", package_file_path, temp_dir, version, sudo_pwd)
+
+    # TODO - Implement the test !!!
+
+    return
+
+def test_python_linux(package_file_path: str, temp_dir: str, version: str, sudo_pwd: str):
+    # Same as test_python_windows except package_file_path is
+    # the ta-lib-{version}-src.tar.gz
+    #
+    # Installation to be done with autotools "./configure"
+
+    log_params("Linux", package_file_path, temp_dir, version, sudo_pwd)
+
+    # TODO - Implement the test !!!
+
+    return

scripts/package.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 
-# Produces and tests the assets to be released.
+# Produces the assets release candidates in 'dist'.
 #
-# The output depends of the host system.
+# The outputs depend of the host system.
 #
 #    For linux/ubuntu: ta-lib-<version>-src.tar.gz
 #         with contents for doing "./configure; make; sudo make install"
@@ -23,11 +23,11 @@
 #    - Scripts must be run in a "VS Development Command Shell" (for the convenience
 #      of CMake and Ninja be on the Path and already be configured).
 #
-#    (FYI, you can optionally do all this in a Windows VM)
+#    (FYI, all this can optionally be done in a Windows VM)
 #
 # How to change the version?
 #   Edit MAJOR, MINOR, BUILD in src/ta_common/ta_version.c
-#   You do not need to modify other files (this script will update all needed files).
+#   There is no need to modify other files (they will be updated by this script).
 
 import filecmp
 import os
@@ -40,7 +40,7 @@
 import zipfile
 import zlib
 
-from common import verify_git_repo, get_version_string, verify_src_package
+from scripts.utilities.common import verify_git_repo, get_version_string, verify_src_package
 
 def compare_zip_files(zip_file1, zip_file2):
     # Does a binary comparison of the contents of the two zip files.
@@ -50,13 +50,13 @@ def compare_zip_files(zip_file1, zip_file2):
         temp_extract_path2 = os.path.join(temp_extract_dir, 'temp2')
         os.makedirs(temp_extract_path1, exist_ok=True)
         os.makedirs(temp_extract_path2, exist_ok=True)
-        
+
         with zipfile.ZipFile(zip_file1, 'r') as zip_ref:
             zip_ref.extractall(temp_extract_path1)
-        
+
         with zipfile.ZipFile(zip_file2, 'r') as zip_ref:
             zip_ref.extractall(temp_extract_path2)
-        
+
         dir_comparison = filecmp.dircmp(temp_extract_path1, temp_extract_path2)
         return not dir_comparison.diff_files and not dir_comparison.left_only and not dir_comparison.right_only
 
@@ -203,15 +203,15 @@ def package_linux(root_dir: str, version: str):
 if __name__ == "__main__":
     root_dir = verify_git_repo()
     version = get_version_string(root_dir)
-
-    if sys.platform == "linux":
+    host_platform = sys.platform
+    if host_platform == "linux":
         package_linux(root_dir,version)
-    elif sys.platform == "win32":
+    elif host_platform == "win32":
         arch = platform.architecture()[0]
         if arch == '64bit':
             package_windows(root_dir, version)
         else:
             print( f"Architecture [{arch}] not yet supported. Only 64 bits supported on windows.")
     else:
-        print("For now, this script is only for Linux systems.")
+        print(f"Unsupported platform [{host_platform}]")
         sys.exit(1)

scripts/test-dist.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+
+# Test release candidate assets in 'dist'
+#
+# This script can be called directly by a dev and also from various
+# GitHub Actions, branches and platforms (e.g. ubuntu-latest, windows-2022)
+#
+# The script must be executed from within a TA-Lib Git repos.
+#
+# Failing this test will **block** the official release.
+#
+# Returns a non-zero exit code if any problem is found.
+
+import argparse
+import os
+import sys
+import platform
+import tempfile
+
+from utilities.common import verify_git_repo, get_version_string, create_temp_dir
+from install_tests.python import test_python_windows, test_python_linux
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Test release candidate assets in 'dist'")
+    parser.add_argument('-p', '--pwd', type=str, default="", help="Password for sudo commands")
+    args = parser.parse_args()
+
+    sudo_pwd = args.pwd
+
+    root_dir = verify_git_repo()
+    version = get_version_string(root_dir)
+    temp_dir = create_temp_dir(root_dir)
+
+    # Identify the dist package to test by this host.
+    host_platform = sys.platform
+    if host_platform == "linux":
+        package_file_path = os.path.join(root_dir, "dist", f"ta-lib-{version}-src.tar.gz")
+    elif host_platform == "win32":
+        arch = platform.architecture()[0]
+        if arch == '64bit':
+            package_file_path = os.path.join(root_dir, "dist", f"ta-lib-{version}-win64.zip")
+        else:
+            print( f"Architecture [{arch}] not yet supported. Only 64 bits supported on windows.")
+    else:
+        print(f"Unsupported platform [{host_platform}]")
+        sys.exit(1)
+
+    if not os.path.isfile(package_file_path):
+        print(f"Package file not found: {package_file_path}. Do './scripts/package.py")
+        sys.exit(1)
+
+    # Simulate user doing a ta-lib-python installation.
+    if host_platform == "linux":
+        test_python_linux(package_file_path, temp_dir, version, sudo_pwd)
+    elif host_platform == "win32":
+        test_python_windows(package_file_path, temp_dir, version, sudo_pwd)
+    else:
+        print(f"Unsupported platform [{host_platform}]")
+        sys.exit(1)

scripts/utilities/__init__.py

@@ -0,0 +1 @@
+# utilities/__init__.py

scripts/common.py → scripts/utilities/common.py

@@ -1,6 +1,28 @@
 import os
+import shutil
 import subprocess
 import sys
+import tempfile
+import time
+
+def create_temp_dir(root_dir) -> str:
+    # Create a temporary directory under root_dir/temp, also purge older ones.
+    #
+    # Return the path of the newly created directory.
+
+    # Delete oldest directories if more than 10 exists and it is more
+    # than 1 hour old.
+    temp_root_dir = os.path.join(root_dir, "temp")
+    os.makedirs(temp_root_dir, exist_ok=True)
+    temp_dirs = sorted(os.listdir(temp_root_dir), key=lambda x: os.path.getctime(os.path.join(temp_root_dir, x)))
+    if len(temp_dirs) > 10:
+        for i in range(len(temp_dirs) - 10):
+            temp_dir_path = os.path.join(temp_root_dir, temp_dirs[i])
+            if os.path.isdir(temp_dir_path) and (time.time() - os.path.getctime(temp_dir_path)) > 3600:
+                shutil.rmtree(temp_dir_path)
+
+    # Create the new temp directory
+    return tempfile.mkdtemp(dir=temp_root_dir)
 
 def verify_git_repo() -> str:
     # Verify that the script is called from within a ta-lib git repos, and if yes