simplify deployment

pyproject.toml

@@ -0,0 +1,6 @@
+
+[build-system]
+    requires = ["setuptools"]
+
+build-backend = "setuptools.build_meta"
+

requirements-ci.txt

@@ -1,15 +1,8 @@
 
-# provided branches for RCT components can be edited,
-# if there is a dependency with non-devel branch
-
-# default RCT branch for CI runs is "devel"
-
-radical.saga @ git+https://github.com/radical-cybertools/radical.saga@devel
-radical.pilot @ git+https://github.com/radical-cybertools/radical.pilot@devel
-
-# RP from the current branch
+# RU from the current branch
 .
 
+requests
 pytest
 pytest-timeout
 pylint

setup.cfg

@@ -2,5 +2,3 @@
 [tool:pytest]
 addopts = -rsxX -v
 
-
-

setup.py

@@ -10,12 +10,10 @@
 
 import re
 import os
-import sys
-import glob
-import shutil
 
 import subprocess as sp
 
+from glob       import glob
 from setuptools import setup, Command, find_namespace_packages
 
 
@@ -24,154 +22,105 @@
 name     = 'radical.%s'      % base
 mod_root = 'src/radical/%s/' % base
 
-# ------------------------------------------------------------------------------
-#
-# pip warning:
-# "In-tree builds are now default. pip 22.1 will enforce this behaviour change.
-#  A possible replacement is to remove the --use-feature=in-tree-build flag."
-#
-# With this change we need to make sure to clean out all temporary files from
-# the src tree. Specifically create (and thus need to clean)
-#   - VERSION
-#   - SDIST
-#   - the sdist file itself (a tarball)
-#
-# `pip install` (or any other direct or indirect invocation of `setup.py`) will
-# in fact run `setup.py` multiple times: one on the top level, and internally
-# again with other arguments to build sdist and bwheel packages.  We must *not*
-# clean out temporary files in those internal runs as that would invalidate the
-# install.
-#
-# We thus introduce an env variable `SDIST_LEVEL` which allows us to separate
-# internal calls from the top level invocation - we only clean on the latter
-# (see end of this file).
-sdist_level = int(os.environ.get('SDIST_LEVEL', 0))
-os.environ['SDIST_LEVEL'] = str(sdist_level + 1)
+scripts  = list(glob('bin/*'))
+root     = os.path.dirname(__file__) or '.'
+descr    = 'Utilities for RADICAL-Cybertools projects'
 
-root = os.path.dirname(__file__) or '.'
+data     = [('share/%s/examples/'    % name, glob('examples/*.{py,cfg}'  )),
+            ('share/%s/examples/zmq' % name, glob('examples/zmq/*.md'    )),
+            ('share/%s/examples/zmq' % name, glob('examples/zmq/queue/*' )),
+            ('share/%s/examples/zmq' % name, glob('examples/zmq/pubsub/*'))]
 
 
 # ------------------------------------------------------------------------------
 #
 def sh_callout(cmd):
-
     p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, shell=True)
-
     stdout, stderr = p.communicate()
     ret            = p.returncode
     return stdout, stderr, ret
 
 
 # ------------------------------------------------------------------------------
 #
-# versioning mechanism:
-#
-#   - version:          1.2.3            - is used for installation
-#   - version_detail:  v1.2.3-9-g0684b06 - is used for debugging
-#   - version is read from VERSION file in root, which then is copied to
-#     module dir, and is getting installed from there.
-#   - version_detail is derived from the git tag, and only available when
-#     installed from git.  That is stored in mod_root/VERSION in the install
-#     tree.
-#   - The VERSION file is used to provide the runtime version information.
-#
 def get_version(_mod_root):
     '''
     a VERSION file containes the version strings is created in mod_root,
     during installation.  That file is used at runtime to get the version
     information.
     '''
 
+    _out = None
+    _err = None
+    _ret = None
     try:
-
+        _version_path   = '%s/%s/VERSION' % (root, _mod_root)
         _version_base   = None
+        _version_short  = None
+        _version_branch = None
+        _version_tag    = None
         _version_detail = None
-        _sdist_name     = None
 
-        # get version from './VERSION'
+        # get `version_base` from distribution's 'VERSION' file
         with open('%s/VERSION' % root, 'r', encoding='utf-8') as fin:
             _version_base = fin.readline().strip()
 
-        # attempt to get version detail information from git
-        # We only do that though if we are in a repo root dir,
-        # ie. if 'git rev-parse --show-prefix' returns an empty string --
-        # otherwise we get confused if the ve lives beneath another repository,
-        # and the pip version used uses an install tmp dir in the ve space
-        # instead of /tmp (which seems to happen with some pip/setuptools
-        # versions).
-        out, _, ret = sh_callout(
-            'cd %s ; '
-            'test -z `git rev-parse --show-prefix` || exit -1; '
-            'tag=`git describe --tags --always` 2>/dev/null ; '
-            'branch=`git branch | grep -e "^*" | cut -f 2- -d " "` 2>/dev/null ; '
-            'echo $tag@$branch' % root)
-        _version_detail = out.strip()
-        _version_detail = _version_detail.decode()
-        _version_detail = _version_detail.replace('detached from ', 'detached-')
-
-        # remove all non-alphanumeric (and then some) chars
-        _version_detail = re.sub('[/ ]+', '-', _version_detail)
-        _version_detail = re.sub('[^[email protected]]+', '', _version_detail)
-
-        if ret              !=  0  or \
-            _version_detail == '@' or \
-            'git-error'      in _version_detail or \
-            'not-a-git-repo' in _version_detail or \
-            'not-found'      in _version_detail or \
-            'fatal'          in _version_detail :
-            _version = _version_base
-        elif '@' not in _version_base:
-            _version = '%s-%s' % (_version_base, _version_detail)
+        _, _, ret = sh_callout('cd %s && git rev-parse --git-dir && which git'
+                               % root)
+        _in_git = (ret == 0)
+
+        if not _in_git:
+
+            with open(_version_path, 'w', encoding='utf-8') as fout:
+                fout.write(_version_base + '\n')
+
         else:
-            _version = _version_base
-
-        # make sure the version files exist for the runtime version inspection
-        _path = '%s/%s' % (root, _mod_root)
-        with open(_path + '/VERSION', 'w', encoding='utf-8') as fout:
-            fout.write(_version_base + '\n')
-            fout.write(_version      + '\n')
-
-        _sdist_name = '%s-%s.tar.gz' % (name, _version_base)
-      # _sdist_name = _sdist_name.replace('/', '-')
-      # _sdist_name = _sdist_name.replace('@', '-')
-      # _sdist_name = _sdist_name.replace('#', '-')
-      # _sdist_name = _sdist_name.replace('_', '-')
-
-        # setuptools 69.5 does changes naming scheme
-        if not os.path.isfile('dist/%s' % _sdist_name):
-            _sdist_name = '%s-%s.tar.gz' % (name.replace('.', '_'), _version_base)
-
-        if os.path.isfile('dist/%s' % _sdist_name):
-            # pip install stage 2 or easy_install stage 1
-            #
-            # pip install will untar the sdist in a tmp tree.  In that tmp
-            # tree, we won't be able to derive git version tags -- so we pack
-            # the formerly derived version as ./VERSION
-            shutil.move('VERSION', 'VERSION.bak')              # backup
-            shutil.copy('%s/VERSION' % _path, 'VERSION')       # version to use
-            os.system  ('python3 setup.py sdist')              # build sdist
-            shutil.copy('dist/%s' % _sdist_name,
-                        '%s/%s'   % (_mod_root, _sdist_name))  # copy into tree
-            shutil.move('VERSION.bak', 'VERSION')              # restore version
-
-        with open(_path + '/SDIST', 'w', encoding='utf-8') as fout:
-            fout.write(_sdist_name + '\n')
-
-        return _version_base, _version_detail, _sdist_name, _path
 
-    except Exception as e:
-        raise RuntimeError('Could not extract/set version: %s' % e) from e
+            # get details from git
+            _out, _, _ret = sh_callout('cd %s && git describe --tags --always' % root)
+            assert _ret == 0, 'git describe failed'
+            _out = _out.decode()
+            _out = _out.strip()
 
+            _version_tag = _out
 
-# ------------------------------------------------------------------------------
-# get version info -- this will create VERSION and srcroot/VERSION
-version, version_detail, sdist_name, path = get_version(mod_root)
+            _out, _, _ret = sh_callout('cd %s && git branch --show-current' % root)
+            assert _ret == 0, 'git branch failed'
+
+            _out = _out.decode()
+            _out = _out.strip()
+
+            _version_branch = _out or 'detached'
+            _version_branch = _version_branch.replace('detached from ', '~')
+
+            _version_short = _version_tag.split('-')[0]
+            _version_short = _version_short[1:]  # strip the 'v'
+
+            if _version_tag:
+                _version_detail = '%s-%s@%s' % (_version_base, _version_tag,
+                                                _version_branch)
+            else:
+                _version_detail = '%s@%s' % (_version_base, _version_branch)
+
+            with open(_version_path, 'w', encoding='utf-8') as fout:
+                fout.write(_version_short  + '\n')
+                fout.write(_version_base   + '\n')
+                fout.write(_version_branch + '\n')
+                fout.write(_version_tag    + '\n')
+                fout.write(_version_detail + '\n')
+
+        return _version_base, _version_path
+
+    except Exception as e:
+        _msg = 'Could not extract/set version: %s' % e
+        if _ret:
+            _msg += '\n' + _out + '\n\n' + _err
+        raise RuntimeError(_msg) from e
 
 
 # ------------------------------------------------------------------------------
-# check python version, should be >= 3.6
-if sys.hexversion < 0x03060000:
-    raise RuntimeError('ERROR: %s requires Python 3.6 or newer' % name)
+# get version info -- this will create VERSION and srcroot/VERSION
+version, version_path = get_version(mod_root)
 
 
 # ------------------------------------------------------------------------------
@@ -181,16 +130,8 @@ class RunTwine(Command):
     def initialize_options(self): pass
     def finalize_options(self):   pass
     def run(self):
-        _, _, ret = sh_callout('python3 setup.py sdist upload -r pypi')
-        raise SystemExit(ret)
-
-
-# ------------------------------------------------------------------------------
-#
-df = [('share/%s/examples/'    % name, glob.glob('examples/*.{py,cfg}'  )),
-      ('share/%s/examples/zmq' % name, glob.glob('examples/zmq/*.md'    )),
-      ('share/%s/examples/zmq' % name, glob.glob('examples/zmq/queue/*' )),
-      ('share/%s/examples/zmq' % name, glob.glob('examples/zmq/pubsub/*'))]
+        _, _, _ret = sh_callout('python3 setup.py sdist upload -r pypi')
+        raise SystemExit(_ret)
 
 
 # ------------------------------------------------------------------------------
@@ -203,9 +144,8 @@ def run(self):
 #
 setup_args = {
     'name'               : name,
-  # 'namespace_packages' : ['radical'],
     'version'            : version,
-    'description'        : 'Utilities for RADICAL-Cybertools projects',
+    'description'        : descr,
     'author'             : 'RADICAL Group at Rutgers University',
     'author_email'       : '[email protected]',
     'maintainer'         : 'The RADICAL Group',
@@ -218,15 +158,15 @@ def run(self):
     },
     'license'            : 'MIT',
     'keywords'           : 'radical utils',
-    'python_requires'    : '>=3.6',
+    'python_requires'    : '>=3.7',
     'classifiers'        : [
         'Development Status :: 5 - Production/Stable',
         'Intended Audience :: Developers',
         'Environment :: Console',
         'License :: OSI Approved :: MIT License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
         'Topic :: Utilities',
         'Topic :: System :: Distributed Computing',
         'Topic :: Scientific/Engineering',
@@ -236,21 +176,12 @@ def run(self):
     ],
     'packages'           : find_namespace_packages('src', include=['radical.*']),
     'package_dir'        : {'': 'src'},
-    'scripts'            : ['bin/radical-utils-fix-headers.pl',
-                            'bin/radical-utils-version',
-                            'bin/radical-utils-pwatch',
-                            'bin/radical-utils-pylint.sh',
-                            'bin/radical-utils-env.sh',
-                            'bin/radical-bridge',
-                            'bin/radical-stack',
-                            'bin/ru.json.sh',
-                            'tests/bin/ru-runcheck.sh',
-                           ],
+    'scripts'            : scripts,
     'package_data'       : {'': ['*.txt', '*.sh', '*.json', '*.gz', '*.c',
-                                 '*.md', 'VERSION', 'SDIST', sdist_name]},
+                                 '*.md', 'VERSION']},
     'install_requires'   : requirements,
     'zip_safe'           : False,
-    'data_files'         : df,
+    'data_files'         : data,
     'cmdclass'           : {'upload': RunTwine},
 }
 
@@ -262,11 +193,8 @@ def run(self):
 
 # ------------------------------------------------------------------------------
 # clean temporary files from source tree
-if sdist_level == 0:
-    os.system('rm -vrf src/%s.egg-info' % name)
-    os.system('rm -vf  %s/%s'           % (path, sdist_name))
-    os.system('rm -vf  %s/VERSION'      % path)
-    os.system('rm -vf  %s/SDIST'        % path)
+os.system('rm -vrf src/%s.egg-info' % name)
+os.system('rm -vf  %s/VERSION'      % version_path)
 
 
 # ------------------------------------------------------------------------------

src/radical/utils/__init__.py

@@ -69,7 +69,7 @@
 from .profile        import PROF_KEY_MAX
 
 from .json_io        import read_json, read_json_str, write_json
-from .json_io        import parse_json, parse_json_str, metric_expand
+from .json_io        import parse_json, parse_json_str
 from .which          import which
 from .tracer         import trace, untrace
 from .get_version    import get_version
@@ -96,10 +96,10 @@
 
 _mod_root = os.path.dirname (__file__)
 
-version_short, version_detail, version_base, \
-               version_branch, sdist_name,   \
-               sdist_path = get_version(_mod_root)
-version = version_short
+version_short, version_base, version_branch, version_tag, version_detail \
+        =  get_version(_mod_root)
+version      = version_short
+__version__  = version_detail
 
 
 # ------------------------------------------------------------------------------