Skip to content

Commit

Permalink
merge from devel
Browse files Browse the repository at this point in the history
  • Loading branch information
@andre-merzky
andre-merzky committed May 8, 2024
2 parents 4fdd0bc + ce78419 commit a84f19b
Show file tree
Hide file tree
Showing 4 changed files with 113 additions and 149 deletions.
1 change: 1 addition & 0 deletions .github/workflows/python-app.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
. testenv/bin/activate
python -m pip install --upgrade pip setuptools
python -m pip install git+https://github.com/radical-cybertools/radical.utils.git@devel
python -m pip install git+https://github.com/radical-cybertools/radical.gtod.git@devel
python -m pip install git+https://github.com/radical-cybertools/radical.saga.git@devel
python -m pip install git+https://github.com/radical-cybertools/radical.pilot.git@devel
python -m pip install --upgrade .
Expand Down
6 changes: 6 additions & 0 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@

[build-system]
requires = ["setuptools"]

build-backend = "setuptools.build_meta"

231 changes: 86 additions & 145 deletions setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,170 +8,125 @@

''' Setup script, only usable via pip. '''

import re
import os
import sys
import glob
import shutil

import subprocess as sp

from glob import glob
from setuptools import setup, Command, find_namespace_packages


# ------------------------------------------------------------------------------
#
base = 'entk'
name = 'radical.%s' % base
mod_root = 'src/radical/%s/' % base

# ------------------------------------------------------------------------------
#
# pip warning:
# "In-tree builds are now default. pip 22.1 will enforce this behaviour change.
# A possible replacement is to remove the --use-feature=in-tree-build flag."
#
# With this change we need to make sure to clean out all temporary files from
# the src tree. Specifically create (and thus need to clean)
# - VERSION
# - SDIST
# - the sdist file itself (a tarball)
#
# `pip install` (or any other direct or indirect invocation of `setup.py`) will
# in fact run `setup.py` multiple times: one on the top level, and internally
# again with other arguments to build sdist and bwheel packages. We must *not*
# clean out temporary files in those internal runs as that would invalidate the
# install.
#
# We thus introduce an env variable `SDIST_LEVEL` which allows us to separate
# internal calls from the top level invocation - we only clean on the latter
# (see end of this file).
sdist_level = int(os.environ.get('SDIST_LEVEL', 0))
os.environ['SDIST_LEVEL'] = str(sdist_level + 1)

root = os.path.dirname(__file__) or '.'
scripts = list(glob('bin/*'))
root = os.path.dirname(__file__) or '.'
readme = open("%s/README.md" % root, encoding='utf-8').read()
descr = "RADICAL-Cybertools: Ensemble Workflow Execution Toolkit"
keywords = ['radical', 'cybertools', 'utilities', 'ensemble', 'workflow']

share = 'share/%s' % name
data = [('%s/examples/advanced/' % share, glob('examples/advanced/*' )),
('%s/examples/analytics/' % share, glob('examples/analytics/*' )),
('%s/examples/misc/' % share, glob('examples/misc/*' )),
('%s/examples/simple/' % share, glob('examples/simple/*' )),
('%s/examples/tutorial/' % share, glob('examples/tutorial/*' )),
('%s/examples/user_guide/' % share, glob('examples/user_guide/*')),
]


# ------------------------------------------------------------------------------
#
def sh_callout(cmd):

p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, shell=True)

stdout, stderr = p.communicate()
ret = p.returncode
return stdout, stderr, ret


# ------------------------------------------------------------------------------
#
# versioning mechanism:
#
# - version: 1.2.3 - is used for installation
# - version_detail: v1.2.3-9-g0684b06 - is used for debugging
# - version is read from VERSION file in root, which then is copied to
# module dir, and is getting installed from there.
# - version_detail is derived from the git tag, and only available when
# installed from git. That is stored in mod_root/VERSION in the install
# tree.
# - The VERSION file is used to provide the runtime version information.
#
def get_version(_mod_root):
'''
a VERSION file containes the version strings is created in mod_root,
during installation. That file is used at runtime to get the version
information.
'''

_out = None
_err = None
_ret = None
try:

_version_path = '%s/%s/VERSION' % (root, _mod_root)
_version_base = None
_version_short = None
_version_branch = None
_version_tag = None
_version_detail = None
_sdist_name = None

# get version from './VERSION'
# get `version_base` from distribution's 'VERSION' file
with open('%s/VERSION' % root, 'r', encoding='utf-8') as fin:
_version_base = fin.readline().strip()

# attempt to get version detail information from git
# We only do that though if we are in a repo root dir,
# ie. if 'git rev-parse --show-prefix' returns an empty string --
# otherwise we get confused if the ve lives beneath another repository,
# and the pip version used uses an install tmp dir in the ve space
# instead of /tmp (which seems to happen with some pip/setuptools
# versions).
out, _, ret = sh_callout(
'cd %s ; '
'test -z `git rev-parse --show-prefix` || exit -1; '
'tag=`git describe --tags --always` 2>/dev/null ; '
'branch=`git branch | grep -e "^*" | cut -f 2- -d " "` 2>/dev/null ; '
'echo $tag@$branch' % root)
_version_detail = out.strip()
_version_detail = _version_detail.decode()
_version_detail = _version_detail.replace('detached from ', 'detached-')

# remove all non-alphanumeric (and then some) chars
_version_detail = re.sub('[/ ]+', '-', _version_detail)
_version_detail = re.sub('[^[email protected]]+', '', _version_detail)

if ret != 0 or \
_version_detail == '@' or \
'git-error' in _version_detail or \
'not-a-git-repo' in _version_detail or \
'not-found' in _version_detail or \
'fatal' in _version_detail :
_version = _version_base
elif '@' not in _version_base:
_version = '%s-%s' % (_version_base, _version_detail)
_, _, ret = sh_callout('cd %s && git rev-parse --git-dir && which git'
% root)
_in_git = (ret == 0)

if not _in_git:

with open(_version_path, 'w', encoding='utf-8') as fout:
fout.write(_version_base + '\n')

else:
_version = _version_base

# make sure the version files exist for the runtime version inspection
_path = '%s/%s' % (root, _mod_root)
with open(_path + '/VERSION', 'w', encoding='utf-8') as fout:
fout.write(_version_base + '\n')
fout.write(_version + '\n')

_sdist_name = '%s-%s.tar.gz' % (name, _version_base)
# _sdist_name = _sdist_name.replace('/', '-')
# _sdist_name = _sdist_name.replace('@', '-')
# _sdist_name = _sdist_name.replace('#', '-')
# _sdist_name = _sdist_name.replace('_', '-')

# setuptools 69.5 does changes naming scheme
if not os.path.isfile('dist/%s' % _sdist_name):
_sdist_name = '%s-%s.tar.gz' % (name.replace('.', '_'), _version_base)

if os.path.isfile('dist/%s' % _sdist_name):
# pip install stage 2 or easy_install stage 1
#
# pip install will untar the sdist in a tmp tree. In that tmp
# tree, we won't be able to derive git version tags -- so we pack
# the formerly derived version as ./VERSION
shutil.move('VERSION', 'VERSION.bak') # backup
shutil.copy('%s/VERSION' % _path, 'VERSION') # version to use
os.system ('python3 setup.py sdist') # build sdist
shutil.copy('dist/%s' % _sdist_name,
'%s/%s' % (_mod_root, _sdist_name)) # copy into tree
shutil.move('VERSION.bak', 'VERSION') # restore version

with open(_path + '/SDIST', 'w', encoding='utf-8') as fout:
fout.write(_sdist_name + '\n')

return _version_base, _version_detail, _sdist_name, _path

except Exception as e:
raise RuntimeError('Could not extract/set version: %s' % e) from e
# get details from git
_out, _err, _ret = sh_callout('cd %s && git describe --tags --always' % root)
assert _ret == 0, 'git describe failed'
_out = _out.decode()
_out = _out.strip()

_version_tag = _out

# ------------------------------------------------------------------------------
# get version info -- this will create VERSION and srcroot/VERSION
version, version_detail, sdist_name, path = get_version(mod_root)
_out, _err, _ret = sh_callout('cd %s && git branch --show-current' % root)
assert _ret == 0, 'git branch failed'

_out = _out.decode()
_out = _out.strip()

_version_branch = _out or 'detached'
_version_branch = _version_branch.replace('detached from ', '~')

_version_short = _version_tag.split('-')[0]
_version_short = _version_short[1:] # strip the 'v'

if _version_tag:
_version_detail = '%s-%s@%s' % (_version_base, _version_tag,
_version_branch)
else:
_version_detail = '%s@%s' % (_version_base, _version_branch)

with open(_version_path, 'w', encoding='utf-8') as fout:
fout.write(_version_short + '\n')
fout.write(_version_base + '\n')
fout.write(_version_branch + '\n')
fout.write(_version_tag + '\n')
fout.write(_version_detail + '\n')

return _version_base, _version_path

except Exception as e:
_msg = 'Could not extract/set version: %s' % e
if _ret:
_msg += '\n' + _out + '\n\n' + _err
raise RuntimeError(_msg) from e


# ------------------------------------------------------------------------------
# check python version, should be >= 3.6
if sys.hexversion < 0x03060000:
raise RuntimeError('ERROR: %s requires Python 3.6 or newer' % name)
# get version info -- this will create VERSION and srcroot/VERSION
version, version_path = get_version(mod_root)


# ------------------------------------------------------------------------------
Expand All @@ -181,20 +136,8 @@ class RunTwine(Command):
def initialize_options(self): pass
def finalize_options(self): pass
def run(self):
_, _, ret = sh_callout('python3 setup.py sdist upload -r pypi')
raise SystemExit(ret)


# ------------------------------------------------------------------------------
#
# This copies the contents like examples/ dir under sys.prefix/share/$name
# It needs the MANIFEST.in entries to work.
base = 'share/%s' % name
df = [('%s/examples/user_guide/' % base, glob.glob('examples/user_guide/*')),
('%s/examples/simple/' % base, glob.glob('examples/simple/*')),
('%s/examples/advanced/' % base, glob.glob('examples/advanced/*')),
('%s/examples/analytics/' % base, glob.glob('examples/analytics/*'))
]
_, _, _ret = sh_callout('python3 setup.py sdist upload -r pypi')
raise SystemExit(_ret)


# ------------------------------------------------------------------------------
Expand All @@ -207,9 +150,10 @@ def run(self):
#
setup_args = {
'name' : name,
# 'namespace_packages' : ['radical'],
'version' : version,
'description' : "Radical Ensemble Toolkit.",
'description' : descr,
'long_description' : readme,
'long_description_content_type' : 'text/markdown',
'author' : 'RADICAL Group at Rutgers University',
'author_email' : '[email protected]',
'maintainer' : 'The RADICAL Group',
Expand All @@ -221,16 +165,16 @@ def run(self):
'Issues' : 'https://github.com/radical-cybertools/%s/issues' % name,
},
'license' : 'MIT',
'keywords' : "ensemble workflow execution",
'python_requires' : '>=3.6',
'keywords' : keywords,
'python_requires' : '>=3.7',
'classifiers' : [
'Development Status :: 5 - Production/Stable',
'Intended Audience :: Developers',
'Environment :: Console',
'License :: OSI Approved :: MIT License',
'Programming Language :: Python',
'Programming Language :: Python :: 3',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Topic :: Utilities',
'Topic :: System :: Distributed Computing',
'Topic :: Scientific/Engineering',
Expand All @@ -240,12 +184,12 @@ def run(self):
],
'packages' : find_namespace_packages('src', include=['radical.*']),
'package_dir' : {'': 'src'},
'scripts' : ['bin/radical-entk-version'],
'scripts' : scripts,
'package_data' : {'': ['*.txt', '*.sh', '*.json', '*.gz', '*.c',
'*.md', 'VERSION', 'SDIST', sdist_name]},
'*.md', 'VERSION']},
'install_requires' : requirements,
'zip_safe' : False,
'data_files' : df,
'data_files' : data,
'cmdclass' : {'upload': RunTwine},
}

Expand All @@ -257,11 +201,8 @@ def run(self):

# ------------------------------------------------------------------------------
# clean temporary files from source tree
if sdist_level == 0:
os.system('rm -vrf src/%s.egg-info' % name)
os.system('rm -vf %s/%s' % (path, sdist_name))
os.system('rm -vf %s/VERSION' % path)
os.system('rm -vf %s/SDIST' % path)
os.system('rm -vrf src/%s.egg-info' % name)
os.system('rm -vf %s' % version_path)


# ------------------------------------------------------------------------------
Expand Down
24 changes: 20 additions & 4 deletions src/radical/entk/__init__.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,17 @@

__copyright__ = "Copyright 2013-2024, http://radical.rutgers.edu"
__license__ = "MIT"


# ------------------------------------------------------------------------------
# we *first* import radical.utils, so that the monkeypatching of the logger has
# a chance to kick in before the logging module is pulled by any other 3rd party
# module, and also to monkeypatch `os.fork()` for the `atfork` functionality
#
import os as _os
import radical.utils as _ru


# ------------------------------------------------------------------------------
#
from .appman import AppManager
Expand All @@ -11,11 +24,14 @@

# ------------------------------------------------------------------------------
#
import os as _os
import radical.utils as _ru
# get version info
#
_mod_root = _os.path.dirname (__file__)

version, version_detail, version_base, version_branch, sdist_name, sdist_path =\
_ru.get_version(_os.path.dirname(__file__))
version_short, version_base, version_branch, version_tag, version_detail \
= _ru.get_version(_mod_root)
version = version_short
__version__ = version_detail


# ------------------------------------------------------------------------------
Expand Down

0 comments on commit a84f19b

Please sign in to comment.