Skip to content

Commit

Permalink
readme
Browse files Browse the repository at this point in the history
  • Loading branch information
@jbae11
jbae11 committed Dec 10, 2020
1 parent 3221c29 commit cf59527
Show file tree
Hide file tree
Showing 3 changed files with 229 additions and 55 deletions.
60 changes: 60 additions & 0 deletions workbench_scripts/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
# Cyclus - Workbench Integration

You can download workbench from [the ORNL gitlab](https://code.ornl.gov/neams-workbench/downloads/-/tree/master).

## Known bugs

1. If you run on python3, and errors when you run `generate_cyclus_sch.py` on `xml2obj`, try with python 2.

## Steps

1. Go into `generate_cyclus_sch.py` and go to the very bottom, modify `path` to your workbench rte path.
2. Run `python generate_cyclus_sch.py`. This script will:
- Try to get the metadata from your Cyclus installation (Running `cyclus -m`)
- If this fails, it will just use the pre-shipped file (`m.json`)
- If you have it set up differently, change the `cyclus_cmd` variable.
- It will generate the following files into your respective workbench directories:
- Grammar file (`cyclus.wbg`)
- This file tells Workbench what files are needed
- and what syntax the input helper would use
- Schema file (`cyclus.sch`)
- schema file has all the rules for syntax validation
- and template pointers
- Template file
- Every archetype gets a template, so it's easier for users to know what attributes each has.
- It also comes with the docstring
- Highlight file
- This file defines different coloring (e.g. for comments, brackets etc.)
- Cyclus runner (`cyclus.py`)
- The runner is the part that `talks` to Workbench
- The runner has modules for taking in executables,
- running cyclus remotely / locally
- Converting files from SON to JSON
- Cyclus processor (`cyclus_processor.py`, `cyclus.wbp`)
- This is not used, due to some bugs.
3. Open Workbench, click on `file -> configurations`.
4. On the very top row, click `Add..` to add Cyclus
5. Configure your executable path, and if you're using a remote server to run your jobs, fill out the remote server address, username, and password
6. Create a new file, with extension `.cyclus`
7. Note that the text format will not be acceptable by Cyclus. `cyclus.py` will convert the SON file format to JSON readable for Cyclus.
8. If not already, click the dropdown next to `Processors` and choose Cyclus.
9. On the text editor, press `control + space` (for mac, other OSs look at `Edit -> Autocomplete` for shortcut)
10. click `simulation`, and the initial template will (hopefully) take it from there.
11. Notice the `validation` tab on the bottom, it will tell you if there's something wrong.
12. Note that if you have validation errors in the block you've been working on, Autocomplete won't work elsewhere.
13. Once finished, click `Run`. That will do:
- convert the current SON file you have to a JSON, and clean it up (outputs as `[your_filename].json`)
- if you defined a remote address:
- uploads the .json file into `/home/[username]/[some_hash]/input.json`
- runs cyclus
- downloads the output to `[your_filename].sqlite`
- if you didn't:
- runs cyclus by `[your_executable_command] [your_filename].json -o [your_filename].sqlite`

## PS

The postprocessor was written, and it does:
- Reads the .sqlite file and generates a long .csv file with 'important metrics' such as material flow
- Workbench can read those .csv files (but not .sqlite files, thus the conversion) and display them as plots and tables.

It is currently commented out in `cyclus.py` (in function `postrun`), but feel free to play with it and see if it's worth it. I'd encourage eventual integration of Cymetric, obviously.
150 changes: 111 additions & 39 deletions workbench_scripts/cyclus.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,11 @@
import sys
import tempfile
import threading
here = os.path.dirname(os.path.abspath(__file__))

sys.path.append(os.path.join(here, 'cyclus'))
#from cyclus_processor import CyclusPostrunner


def unpack_stringlist(stringlist):
"""parses stringlist that was formatted to pass on command-line into original array of strings"""
Expand Down Expand Up @@ -235,7 +240,7 @@ def __add_options(self):
"type": "stringlist"
})
shared.append({
"default": self.executable,
"default": "cyclus", #self.executable,
"dest": "executable",
"flag": "-e",
"help": "Path to the executable to run",
Expand Down Expand Up @@ -530,6 +535,10 @@ def output_directory_overridden(self):
return self.output_directory != None

def postrun(self, options):
# something Cymetric, I suppose
# self.echo(1, '#### Postrunner on %s' %self.output_path)
# CyclusPostrunner(self.output_path)
# self.echo(1, '#### Finished Postrunner.')
"""actions to perform after the run finishes"""

def prerun(self, options):
Expand All @@ -547,6 +556,21 @@ def prerun(self, options):
self.echo(2, "# ", options.working_directory)
os.chdir(options.working_directory)


# convert son to JSON and clean the JSON
binpath = os.path.join(here, os.pardir, 'bin')
sonjson_path = os.path.join(binpath, 'sonjson')
self.echo(1, '#### Converting SON to JSON.... ')
schema_file_path = os.path.join(here, 'cyclus', 'cyclus.sch')
p = subprocess.Popen([sonjson_path, schema_file_path, options.input],
stdout=subprocess.PIPE)
json_str = p.stdout.read()
extension = options.input.split('.')[-1]
self.json_filepath = options.input.replace(extension, 'json')
temp_json_path = os.path.join(self.working_directory, self.json_filepath)
with open(temp_json_path, 'w') as f:
f.write(self.clean_json(json_str))
self.echo(1, '#### Finished converting to JSON! ')
self.echo(1, "#### Pre-run ####")
self.echo(1)

Expand Down Expand Up @@ -590,17 +614,33 @@ def process_args(self, parser, args):

self.echo(1, '# Connected.')
self.echo(1, '# Testing if the executable exists...')
self.echo(1, '# Running "command -v %s"' %self.executable)
# check if file is executable
output = self.remote_execute('test -x %s && echo "yayyy"' %self.executable)
if 'yay' in output:
self.echo(1, '# The file in the defined path is executable.')
output = self.remote_execute('command -v ' + self.executable)
if not output:
self.echo(0, '# The executable does not seem to exist...')
self.echo(0, '# Let me try something here and try to see if it is defined in bashrc...')
out = self.remote_execute('cat ~/.bashrc | grep "alias %s"' %self.executable)
if len(out) != 0:
self.echo(0, '# We found \n%s\n.. gonna try if this works' %out)
potential_cmd = out.strip().split('\n')[-1].split('=')[-1].strip()
out2 = self.remote_execute(potential_cmd)
if 'No input file' in out2:
self.echo(0, '# seems to have worked. Gonna Switch executable to %s' %potential_cmd)
self.executable = potential_cmd
else:
self.echo(0, '# Could not find executable path. Try the full path to the executable instead of an alias or command.')
sys.exit(1)
else:
self.echo(0, '# Could not find executable path. Try the full path to the executable instead of an alias or command.')
sys.exit(1)
else:
self.echo(0, 'The file is not Executable')
sys.exit(1)
self.echo(1, 'The executable is good!')

except Exception as e:
self.echo(0, 'Could not connect. Check arguments.')
self.echo(0, 'See Error below:')
self.echo(0, e)
print(e)
sys.exit(1)

else:
Expand Down Expand Up @@ -669,6 +709,8 @@ def process_args(self, parser, args):
self.echo(1)

def remote_execute(self, cmd):
self.echo(1, '## Remotely executing command:')
self.echo(1, cmd)
i, o, e = self.ssh.exec_command(cmd)
output = '\n'.join(o.readlines())
error = '\n'.join(e.readlines())
Expand All @@ -677,6 +719,33 @@ def remote_execute(self, cmd):
return output


def clean_json(self, s):
news = []
was_value = False
#was_null = False
for indx, line in enumerate(s.split('\n')):
if '"null"' in line:
line = line.replace('"null"', r'{}')
#if was_null:
# line = line.replace(']', '')
if was_value:
line = line.replace('}', '')
was_value = False
if '"value"' in line:
news[-1] = news[-1].replace('{', '')
news.append(line.replace('"value":', ''))
was_value = True
#if '"null"' in line:
# news[-1] = news[-1].replace('[', '')
# news.append(line.replace('"null"', r'{}'))
# was_null = True
else:
news.append(line)
parsed = json.loads(''.join(news))

return json.dumps(parsed, indent=4, sort_keys=True)


def run(self, options):
"""run the given executable"""
self.echo(1, "#### Run ", self.app_name(), " ####")
Expand All @@ -686,7 +755,6 @@ def run(self, options):
# Workbench's python environment as this is likely more
# recent and contains packages that are not available
# with default Python installations
print(self.executable)
if self.executable.endswith(".py"):
args = [sys.executable, self.executable]
else:
Expand All @@ -697,8 +765,7 @@ def run(self, options):
args.extend(self.additional)
# request list of supported arguments to pass to the executable
args.extend(self.run_args(options))
print('args')
print(args)
self.output_path = self.json_filepath.replace('.json', '.sqlite')
if self.is_remote:
self.echo(1, "#### Executing '", " ".join(args), "' on remote server %s " %self.remote_server_address)
rtncode = 0
Expand All @@ -712,44 +779,46 @@ def run(self, options):
import os
while duplicate_hash and n < 3:
rnd_dir = os.path.join('/home/', self.remote_server_username, str(uuid.uuid4()))
remote_input_path = os.path.join(rnd_dir, 'input.xml')
remote_output_path = remote_input_path.replace('.xml', '.sqlite')
remote_input_path = os.path.join(rnd_dir, 'input.json')
remote_output_path = remote_input_path.replace('.json', '.sqlite')
output = self.remote_execute('mkdir %s' %rnd_dir)
print('error', output)
n+=1
if not output:
# empty output means nothing went wrong,
duplicate_hash = False
self.echo(1, '# Uploading input file to %s' %self.remote_server_address)
self.echo(1, '# To path "%s"' %remote_input_path)
ftp = self.ssh.open_sftp()
ftp.put(options.input ,remote_input_path)
ftp.put(self.json_filepath ,remote_input_path)

self.echo(1, '# Now running %s...' %self.app_name())
output = self.remote_execute('%s %s -o %s --warn-limit 0' %(self.executable, remote_input_path, remote_output_path))
cmd = '%s %s -o %s --warn-limit 0' %(self.executable, remote_input_path, remote_output_path)
self.echo(1, '# Running command: %s' %cmd)
output = self.remote_execute(cmd)
# this is super wonky, consider changing
if output == 0 or ('Error' not in output and 'error' not in output and 'Abort' not in output and 'fatal' not in output and 'Invalid' not in output):
if output == 0 or ('ERROR' not in output.upper() and 'ABORT' not in output.upper() and 'FATAL' not in output.upper() and 'INVALID' not in output.upper() and 'FAILED TO VALIDATE' not in output.upper()):

self.echo(1, '############################' )
self.echo(1, '# %s ran successfully!' %self.app_name())
self.echo(1, '############################' )

self.echo(1, '# Now downloading output file')
pre, ext = os.path.splitext(options.input)
ftp.get(remote_output_path, os.path.join(self.working_directory, pre + '.out'))

self.pre = pre
ftp.get(remote_output_path, self.output_path)
# this is super wonky, consider changing
time.sleep(5)
self.echo(1, '# Download complete (%s)' %os.path.join(self.working_directory, pre + '.out'))
time.sleep(10)
self.echo(1, '# Download complete (%s)' %self.output_path)

else:
self.echo(1, '# Run Failed! See the following output')
self.echo(1, output)

except Exception as e:
self.echo(0, 'Something Went Wrong')
self.echo(0, 'Something went wrong during running Cyclus remotely:')
self.echo(0, 'See Error below:')
print(e)
self.echo(0, e)
sys.exit(1)

else:
Expand All @@ -758,24 +827,27 @@ def run(self, options):
# execute
rtncode = 0
try:
proc = subprocess.Popen(args, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)

# tee-output objects
teeout = None
teeerr = None

# tee requested
if self.tee:
teeout = open(options.output_basename + ".out", "w")
teeerr = open(options.output_basename + ".err", "w")

# start background readers
out = threading.Thread(target=streamer, name="out_reader",
args=(proc.stdout, sys.stdout, teeout))
err = threading.Thread(target=streamer, name="err_reader",
args=(proc.stderr, sys.stderr, teeerr))
out.start()
err.start()
args = [self.executable, self.json_filepath, '-o', self.output_path]
proc = subprocess.Popen(args, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.PIPE)


if False:
# tee-output objects
teeout = None
teeerr = None

# tee requested
if self.tee:
teeout = open(options.output_basename + ".out", "w")
teeerr = open(options.output_basename + ".err", "w")

# start background readers
out = threading.Thread(target=streamer, name="out_reader",
args=(proc.stdout, sys.stdout, teeout))
err = threading.Thread(target=streamer, name="err_reader",
args=(proc.stderr, sys.stderr, teeerr))
out.start()
err.start()

# wait for process to finish
proc.wait()
Expand Down
Loading

0 comments on commit cf59527

Please sign in to comment.