removes popper init (#615)

  * removes popper init as we do not rely on .popper.yml file anymore
  * removes docker calls from dry-run test
  * cleanup for 'popper dot'
  * info and action_info levels of logger are now printed to stdout. Everything else to stderr
  * adds code for dealing with unreachable actions
  * adds fix for unreachable actions in scaffolded workflow in 'test scaffold'
  * improves some messages

fixes #611

.travis.yml

@@ -10,7 +10,7 @@ before_install:
 - wget http://neurodeb.pirsquared.org/pool/main/s/singularity-container/singularity-container_2.6.1-2~nd16.04+1_amd64.deb
 - sudo apt-get -f install ./singularity-container_2.6.1-2~nd16.04+1_amd64.deb
 - pip install git+https://github.com/virtuald/[email protected]#egg=pyhcl
-script:
+install:
 - pip install cli/
-- which popper
+script:
 - popper run --recursive

ci/cli.workflow

@@ -2,90 +2,85 @@ workflow "cli tests" {
   on = "push"
   resolves = "end"
 }
-action "Shellcheck" {
+action "lint" {
   uses = "actions/bin/shellcheck@master"
   args = "-x ./ci/test/*"
 }
-action "test init" {
-  needs = "Shellcheck"
-  uses = "./ci/test"
-  runs = "init"
-}
 
 action "test ci" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "ci"
 }
 
 action "test reuse" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "reuse"
 }
 
 action "test actions-demo" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "actions-demo"
 }
 
 action "test validate" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "validate"
 }
 
 action "test scaffold" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "scaffold"
 }
 
 action "test recursive" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "recursive"
 }
 
 action "test dry-run" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "dry-run"
 }
 
 action "test parallel" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "parallel"
 }
 
 action "test dot" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "dot"
 }
 
 action "test singularity" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "singularity"
 }
 
 action "test interrupt" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "interrupt"
 }
 
 action "test add" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "add"
 }
 
 action "test quiet" {
-  needs = "Shellcheck"
+  needs = "lint"
   uses = "./ci/test"
   runs = "quiet"
 }
@@ -99,7 +94,6 @@ action "end" {
     "test ci",
     "test dot",
     "test dry-run",
-    "test init",
     "test interrupt",
     "test parallel",
     "test quiet",

ci/test/actions-demo

@@ -16,21 +16,30 @@ pushd /tmp/npm
 DOCKER_USERNAME=foo DOCKER_PASSWORD=bar popper run "Integration Test"
 popd
 
-# test with actions hosted on gitlab
+# test with a remote action defined on the root of the repo
 init_test_repo
 cd $test_repo_path
 cat <<EOF > main.workflow
-workflow "clone" {
-  resolves = "test clone git@"
+workflow "action on root folder" {
+  resolves = "test"
 }
-
-action "test clone https" {
-  uses = "https://gitlab.com/barbaragd/action-test.git@master"
+action "test" {
+  uses = "popperized/ansible@master"
+  args = "--version"
+  secrets = ["ANSIBLE_SSH_KEY_DATA"]
 }
+EOF
+
+ANSIBLE_SSH_KEY_DATA="$(echo 'dummydata' | base64)" popper run
 
-action "test clone git@"{
-  uses = "[email protected]:barbaragd/action-test.git@master"
-  needs = "test clone https"
+# test with actions hosted on gitlab
+cd $test_repo_path
+cat <<EOF > main.workflow
+workflow "clone" {
+  resolves = "test"
+}
+action "test" {
+  uses = "https://gitlab.com/barbaragd/action-test@master"
 }
 EOF
 

ci/test/ci

@@ -5,7 +5,6 @@ source ./common
 init_test_repo
 
 cd "$test_repo_path"
-popper init
 
 popper ci --service travis
 test -f .travis.yml

ci/test/dry-run

@@ -13,18 +13,11 @@ git checkout c824a5a
 
 export PHONY_SECRET=foo
 
-# delete any containers that might exist
-mapfile -t args < <(docker ps -aq)
-docker rm "${args[@]}" || true
-
 dry_num_lines=$(popper run --dry-run | grep -c 'DRYRUN')
 
 # 5 actions, at least one line per action
 test "$dry_num_lines" -gt 5
 
-# test that no containers are running
-test -z "$(docker ps -aq)"
-
 ## `--dry-run` test for SingularityRunner
 
 cd "$test_repo_path"

ci/test/scaffold

@@ -7,25 +7,19 @@ cd "$test_repo_path"
 
 popper scaffold
 
-test -f $test_repo_path/main.workflow
-test -f $test_repo_path/actions/example/Dockerfile
-test -f $test_repo_path/actions/example/entrypoint.sh
-test -f $test_repo_path/actions/example/README.md
+test -f main.workflow
+test -f actions/example/Dockerfile
+test -f actions/example/entrypoint.sh
+test -f actions/example/README.md
 
 popper run
 
-rm main.workflow
-
-mkdir pipelines
-cd pipelines
-mkdir mypipe
-cd mypipe
+mkdir -p pipelines/mypipe
+cd pipelines/mypipe
 
 popper scaffold
 
-cd ../..
-
-test -f $test_repo_path/pipelines/mypipe/main.workflow
-test -f $test_repo_path/pipelines/mypipe/actions/example/Dockerfile
-test -f $test_repo_path/pipelines/mypipe/actions/example/entrypoint.sh
-test -f $test_repo_path/pipelines/mypipe/actions/example/README.md
+test -f main.workflow
+test -f actions/example/Dockerfile
+test -f actions/example/entrypoint.sh
+test -f actions/example/README.md

ci/test/validate

@@ -411,3 +411,22 @@ action "show env" {
 EOF
 
 popper run
+
+## show WARNING on unreachable actions
+cat <<EOF > main.workflow
+workflow "test" {
+  resolves = "reachable"
+}
+
+action "reachable" {
+  uses = "actions/bin/sh@master"
+  args = "ls"
+}
+
+action "unreachable" {
+  uses = "actions/bin/sh@master"
+  args = "ls"
+}
+EOF
+
+popper run 2>&1 | grep 'WARNING'

cli/popper/cli.py

@@ -8,26 +8,12 @@
 
 from popper import __version__ as popper_version, log as log
 
-log = log.setup_logging()
 
 class Context(object):
-
-    def __init__(self):
-        self.verbose = False
-        self.home = os.getcwd()
-
-    def log(self, msg, *args):
-        """Logs a message to stderr."""
-        if args:
-            msg %= args
-        click.echo(msg, file=sys.stderr)
-
-    def vlog(self, msg, *args):
-        """Logs a message to stderr only if verbose is enabled."""
-        if self.verbose:
-            self.log(msg, *args)
+    pass
 
 
+log = log.setup_logging()
 pass_context = click.make_pass_decorator(Context, ensure=True)
 cmd_folder = os.path.abspath(os.path.join(os.path.dirname(__file__),
                                           'commands'))
@@ -97,6 +83,6 @@ def signal_handler(sig, frame):
 
     for container in docker_list:
         log.info("Stopping container '{}'".format(container.name))
-        container.stop()
+        container.stop(timeout=1)
 
     sys.exit(0)

cli/popper/commands/cmd_ci.py

@@ -86,12 +86,18 @@
 )
 @pass_context
 def cli(ctx, service):
-    """Generates configuration files for distinct CI services.
+    """Generates configuration files for distinct CI services. This command
+    needs to be executed on the root of your Git repository folder.
     """
     if service not in ci_files:
         log.fail("Unrecognized service " + service)
 
-    project_root = scm.get_popper_root_folder()
+    project_root = scm.get_git_root_folder()
+
+    if project_root != os.getcwd():
+        log.fail(
+            'This command needs to be executed on the root of your '
+            'Git project folder (where the .git/ folder is located).')
 
     for ci_file, ci_file_content in pu.get_items(ci_files[service]):
         ci_file_content = ci_file_content

cli/popper/commands/cmd_dot.py

@@ -21,62 +21,34 @@
     required=False,
     is_flag=True
 )
-@click.command('dot', short_help='Generate a graph in the .dot format')
+@click.command('dot', short_help='Generate a graph in the .dot format.')
 @pass_context
 def cli(ctx, wfile, recursive):
     """
-    Creates a graph in the .dot format representing the workflow
+    Creates a graph in the .dot format representing the workflow.
     """
+    def add_to_graph(graph_str, wf, parent, children):
+        """Recursively goes through "next" and adds corresponding actions
+        """
+        _parent = parent.replace(' ', '_').replace('-', '_')
+        for n in children:
+            _n = n.replace(' ', '_').replace('-', '_')
+            graph_str += "  {} -> {};\n".format(_parent, _n)
+            for M in wf['action'][n].get('next', []):
+                graph_str = add_to_graph(graph_str, wf, n, [M])
+
+        return graph_str
+
     wfile_list = list()
+
     if recursive:
         wfile_list = pu.find_recursive_wfile()
     else:
         wfile_list.append(pu.find_default_wfile(wfile))
 
     for wfile in wfile_list:
         pipeline = Workflow(wfile, False, False, False, False, True)
-
-        graph = list()
-
         wf = pipeline.wf
-        workflow_name = list(wf['workflow'].keys())[0]
-
-        action = wf['resolves'][0]
-        last_action = get_first_action(wf)
-
-        for act in last_action:
-            graph.append("\t{} -> {};\n".format(
-                workflow_name.replace(' ', '_').replace('-', '_'),
-                act.replace(' ', '_').replace('-', '_')))
-
-        parent_action = cur_action = action
-        graph = add(parent_action, cur_action, wf['action'], graph)
-        graph = ''.join(list(set(graph)))
-        graph = "digraph G {\n" + graph + "}\n"
-        log.info(graph)
-
-
-# Recursively go through "needs" and add corresponding actions to graph
-def add(parent_action, cur_action, actions, graph):
-
-    if 'needs' in actions[cur_action]:
-        action_list = actions[cur_action]['needs']
-        for act in action_list:
-            graph = add(cur_action, act, actions, graph)
-
-    # Adds edges to the graph
-    if cur_action != parent_action:
-        graph.append("\t{} -> {};\n".format(
-            cur_action.replace(' ', '_').replace('-', '_'),
-            parent_action.replace(' ', '_').replace('-', '_')))
-
-    return graph
-
-
-def get_first_action(wf):
-    actions = list()
-    for act in wf['action']:
-        if act in wf['action']:
-            if 'needs' not in wf['action'][act]:
-                actions.append(act)
-    return actions
+        workflow_name = wf['name'].replace(' ', '_').replace('-', '_')
+        graph_str = add_to_graph("", wf, workflow_name, wf['root'])
+        log.info("digraph G {\n" + graph_str + "}\n")