diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 99fd78de..e4474e49 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -27,6 +27,7 @@ jobs: . testenv/bin/activate python -m pip install --upgrade pip setuptools python -m pip install git+https://github.com/radical-cybertools/radical.utils.git@devel + python -m pip install git+https://github.com/radical-cybertools/radical.gtod.git@devel python -m pip install git+https://github.com/radical-cybertools/radical.saga.git@devel python -m pip install git+https://github.com/radical-cybertools/radical.pilot.git@devel python -m pip install --upgrade . diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..10dca9a2 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,6 @@ + +[build-system] + requires = ["setuptools"] + +build-backend = "setuptools.build_meta" + diff --git a/setup.py b/setup.py index 97233eb8..b2e00d28 100755 --- a/setup.py +++ b/setup.py @@ -8,55 +8,40 @@ ''' Setup script, only usable via pip. ''' -import re import os -import sys -import glob -import shutil import subprocess as sp +from glob import glob from setuptools import setup, Command, find_namespace_packages # ------------------------------------------------------------------------------ +# base = 'entk' name = 'radical.%s' % base mod_root = 'src/radical/%s/' % base -# ------------------------------------------------------------------------------ -# -# pip warning: -# "In-tree builds are now default. pip 22.1 will enforce this behaviour change. -# A possible replacement is to remove the --use-feature=in-tree-build flag." -# -# With this change we need to make sure to clean out all temporary files from -# the src tree. Specifically create (and thus need to clean) -# - VERSION -# - SDIST -# - the sdist file itself (a tarball) -# -# `pip install` (or any other direct or indirect invocation of `setup.py`) will -# in fact run `setup.py` multiple times: one on the top level, and internally -# again with other arguments to build sdist and bwheel packages. We must *not* -# clean out temporary files in those internal runs as that would invalidate the -# install. -# -# We thus introduce an env variable `SDIST_LEVEL` which allows us to separate -# internal calls from the top level invocation - we only clean on the latter -# (see end of this file). -sdist_level = int(os.environ.get('SDIST_LEVEL', 0)) -os.environ['SDIST_LEVEL'] = str(sdist_level + 1) - -root = os.path.dirname(__file__) or '.' +scripts = list(glob('bin/*')) +root = os.path.dirname(__file__) or '.' +readme = open("%s/README.md" % root, encoding='utf-8').read() +descr = "RADICAL-Cybertools: Ensemble Workflow Execution Toolkit" +keywords = ['radical', 'cybertools', 'utilities', 'ensemble', 'workflow'] + +share = 'share/%s' % name +data = [('%s/examples/advanced/' % share, glob('examples/advanced/*' )), + ('%s/examples/analytics/' % share, glob('examples/analytics/*' )), + ('%s/examples/misc/' % share, glob('examples/misc/*' )), + ('%s/examples/simple/' % share, glob('examples/simple/*' )), + ('%s/examples/tutorial/' % share, glob('examples/tutorial/*' )), + ('%s/examples/user_guide/' % share, glob('examples/user_guide/*')), +] # ------------------------------------------------------------------------------ # def sh_callout(cmd): - p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, shell=True) - stdout, stderr = p.communicate() ret = p.returncode return stdout, stderr, ret @@ -64,17 +49,6 @@ def sh_callout(cmd): # ------------------------------------------------------------------------------ # -# versioning mechanism: -# -# - version: 1.2.3 - is used for installation -# - version_detail: v1.2.3-9-g0684b06 - is used for debugging -# - version is read from VERSION file in root, which then is copied to -# module dir, and is getting installed from there. -# - version_detail is derived from the git tag, and only available when -# installed from git. That is stored in mod_root/VERSION in the install -# tree. -# - The VERSION file is used to provide the runtime version information. -# def get_version(_mod_root): ''' a VERSION file containes the version strings is created in mod_root, @@ -82,96 +56,77 @@ def get_version(_mod_root): information. ''' + _out = None + _err = None + _ret = None try: - + _version_path = '%s/%s/VERSION' % (root, _mod_root) _version_base = None + _version_short = None + _version_branch = None + _version_tag = None _version_detail = None - _sdist_name = None - # get version from './VERSION' + # get `version_base` from distribution's 'VERSION' file with open('%s/VERSION' % root, 'r', encoding='utf-8') as fin: _version_base = fin.readline().strip() - # attempt to get version detail information from git - # We only do that though if we are in a repo root dir, - # ie. if 'git rev-parse --show-prefix' returns an empty string -- - # otherwise we get confused if the ve lives beneath another repository, - # and the pip version used uses an install tmp dir in the ve space - # instead of /tmp (which seems to happen with some pip/setuptools - # versions). - out, _, ret = sh_callout( - 'cd %s ; ' - 'test -z `git rev-parse --show-prefix` || exit -1; ' - 'tag=`git describe --tags --always` 2>/dev/null ; ' - 'branch=`git branch | grep -e "^*" | cut -f 2- -d " "` 2>/dev/null ; ' - 'echo $tag@$branch' % root) - _version_detail = out.strip() - _version_detail = _version_detail.decode() - _version_detail = _version_detail.replace('detached from ', 'detached-') - - # remove all non-alphanumeric (and then some) chars - _version_detail = re.sub('[/ ]+', '-', _version_detail) - _version_detail = re.sub('[^a-zA-Z0-9_+@.-]+', '', _version_detail) - - if ret != 0 or \ - _version_detail == '@' or \ - 'git-error' in _version_detail or \ - 'not-a-git-repo' in _version_detail or \ - 'not-found' in _version_detail or \ - 'fatal' in _version_detail : - _version = _version_base - elif '@' not in _version_base: - _version = '%s-%s' % (_version_base, _version_detail) + _, _, ret = sh_callout('cd %s && git rev-parse --git-dir && which git' + % root) + _in_git = (ret == 0) + + if not _in_git: + + with open(_version_path, 'w', encoding='utf-8') as fout: + fout.write(_version_base + '\n') + else: - _version = _version_base - - # make sure the version files exist for the runtime version inspection - _path = '%s/%s' % (root, _mod_root) - with open(_path + '/VERSION', 'w', encoding='utf-8') as fout: - fout.write(_version_base + '\n') - fout.write(_version + '\n') - - _sdist_name = '%s-%s.tar.gz' % (name, _version_base) - # _sdist_name = _sdist_name.replace('/', '-') - # _sdist_name = _sdist_name.replace('@', '-') - # _sdist_name = _sdist_name.replace('#', '-') - # _sdist_name = _sdist_name.replace('_', '-') - - # setuptools 69.5 does changes naming scheme - if not os.path.isfile('dist/%s' % _sdist_name): - _sdist_name = '%s-%s.tar.gz' % (name.replace('.', '_'), _version_base) - - if os.path.isfile('dist/%s' % _sdist_name): - # pip install stage 2 or easy_install stage 1 - # - # pip install will untar the sdist in a tmp tree. In that tmp - # tree, we won't be able to derive git version tags -- so we pack - # the formerly derived version as ./VERSION - shutil.move('VERSION', 'VERSION.bak') # backup - shutil.copy('%s/VERSION' % _path, 'VERSION') # version to use - os.system ('python3 setup.py sdist') # build sdist - shutil.copy('dist/%s' % _sdist_name, - '%s/%s' % (_mod_root, _sdist_name)) # copy into tree - shutil.move('VERSION.bak', 'VERSION') # restore version - - with open(_path + '/SDIST', 'w', encoding='utf-8') as fout: - fout.write(_sdist_name + '\n') - - return _version_base, _version_detail, _sdist_name, _path - except Exception as e: - raise RuntimeError('Could not extract/set version: %s' % e) from e + # get details from git + _out, _err, _ret = sh_callout('cd %s && git describe --tags --always' % root) + assert _ret == 0, 'git describe failed' + _out = _out.decode() + _out = _out.strip() + _version_tag = _out -# ------------------------------------------------------------------------------ -# get version info -- this will create VERSION and srcroot/VERSION -version, version_detail, sdist_name, path = get_version(mod_root) + _out, _err, _ret = sh_callout('cd %s && git branch --show-current' % root) + assert _ret == 0, 'git branch failed' + + _out = _out.decode() + _out = _out.strip() + + _version_branch = _out or 'detached' + _version_branch = _version_branch.replace('detached from ', '~') + + _version_short = _version_tag.split('-')[0] + _version_short = _version_short[1:] # strip the 'v' + + if _version_tag: + _version_detail = '%s-%s@%s' % (_version_base, _version_tag, + _version_branch) + else: + _version_detail = '%s@%s' % (_version_base, _version_branch) + + with open(_version_path, 'w', encoding='utf-8') as fout: + fout.write(_version_short + '\n') + fout.write(_version_base + '\n') + fout.write(_version_branch + '\n') + fout.write(_version_tag + '\n') + fout.write(_version_detail + '\n') + + return _version_base, _version_path + + except Exception as e: + _msg = 'Could not extract/set version: %s' % e + if _ret: + _msg += '\n' + _out + '\n\n' + _err + raise RuntimeError(_msg) from e # ------------------------------------------------------------------------------ -# check python version, should be >= 3.6 -if sys.hexversion < 0x03060000: - raise RuntimeError('ERROR: %s requires Python 3.6 or newer' % name) +# get version info -- this will create VERSION and srcroot/VERSION +version, version_path = get_version(mod_root) # ------------------------------------------------------------------------------ @@ -181,20 +136,8 @@ class RunTwine(Command): def initialize_options(self): pass def finalize_options(self): pass def run(self): - _, _, ret = sh_callout('python3 setup.py sdist upload -r pypi') - raise SystemExit(ret) - - -# ------------------------------------------------------------------------------ -# -# This copies the contents like examples/ dir under sys.prefix/share/$name -# It needs the MANIFEST.in entries to work. -base = 'share/%s' % name -df = [('%s/examples/user_guide/' % base, glob.glob('examples/user_guide/*')), - ('%s/examples/simple/' % base, glob.glob('examples/simple/*')), - ('%s/examples/advanced/' % base, glob.glob('examples/advanced/*')), - ('%s/examples/analytics/' % base, glob.glob('examples/analytics/*')) -] + _, _, _ret = sh_callout('python3 setup.py sdist upload -r pypi') + raise SystemExit(_ret) # ------------------------------------------------------------------------------ @@ -207,9 +150,10 @@ def run(self): # setup_args = { 'name' : name, - # 'namespace_packages' : ['radical'], 'version' : version, - 'description' : "Radical Ensemble Toolkit.", + 'description' : descr, + 'long_description' : readme, + 'long_description_content_type' : 'text/markdown', 'author' : 'RADICAL Group at Rutgers University', 'author_email' : 'radical@rutgers.edu', 'maintainer' : 'The RADICAL Group', @@ -221,8 +165,8 @@ def run(self): 'Issues' : 'https://github.com/radical-cybertools/%s/issues' % name, }, 'license' : 'MIT', - 'keywords' : "ensemble workflow execution", - 'python_requires' : '>=3.6', + 'keywords' : keywords, + 'python_requires' : '>=3.7', 'classifiers' : [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', @@ -230,7 +174,7 @@ def run(self): 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Utilities', 'Topic :: System :: Distributed Computing', 'Topic :: Scientific/Engineering', @@ -240,12 +184,12 @@ def run(self): ], 'packages' : find_namespace_packages('src', include=['radical.*']), 'package_dir' : {'': 'src'}, - 'scripts' : ['bin/radical-entk-version'], + 'scripts' : scripts, 'package_data' : {'': ['*.txt', '*.sh', '*.json', '*.gz', '*.c', - '*.md', 'VERSION', 'SDIST', sdist_name]}, + '*.md', 'VERSION']}, 'install_requires' : requirements, 'zip_safe' : False, - 'data_files' : df, + 'data_files' : data, 'cmdclass' : {'upload': RunTwine}, } @@ -257,11 +201,8 @@ def run(self): # ------------------------------------------------------------------------------ # clean temporary files from source tree -if sdist_level == 0: - os.system('rm -vrf src/%s.egg-info' % name) - os.system('rm -vf %s/%s' % (path, sdist_name)) - os.system('rm -vf %s/VERSION' % path) - os.system('rm -vf %s/SDIST' % path) +os.system('rm -vrf src/%s.egg-info' % name) +os.system('rm -vf %s' % version_path) # ------------------------------------------------------------------------------ diff --git a/src/radical/entk/__init__.py b/src/radical/entk/__init__.py index 7ef16cc2..70ad0e97 100644 --- a/src/radical/entk/__init__.py +++ b/src/radical/entk/__init__.py @@ -1,4 +1,17 @@ +__copyright__ = "Copyright 2013-2024, http://radical.rutgers.edu" +__license__ = "MIT" + + +# ------------------------------------------------------------------------------ +# we *first* import radical.utils, so that the monkeypatching of the logger has +# a chance to kick in before the logging module is pulled by any other 3rd party +# module, and also to monkeypatch `os.fork()` for the `atfork` functionality +# +import os as _os +import radical.utils as _ru + + # ------------------------------------------------------------------------------ # from .appman import AppManager @@ -11,11 +24,14 @@ # ------------------------------------------------------------------------------ # -import os as _os -import radical.utils as _ru +# get version info +# +_mod_root = _os.path.dirname (__file__) -version, version_detail, version_base, version_branch, sdist_name, sdist_path =\ - _ru.get_version(_os.path.dirname(__file__)) +version_short, version_base, version_branch, version_tag, version_detail \ + = _ru.get_version(_mod_root) +version = version_short +__version__ = version_detail # ------------------------------------------------------------------------------