From cf59527413fe21930dc00adb2848bce4b95df1d2 Mon Sep 17 00:00:00 2001 From: Bae Date: Wed, 9 Dec 2020 21:42:40 -0500 Subject: [PATCH] readme --- workbench_scripts/README.md | 60 +++++++++ workbench_scripts/cyclus.py | 150 +++++++++++++++++------ workbench_scripts/generate_cyclus_sch.py | 74 ++++++++--- 3 files changed, 229 insertions(+), 55 deletions(-) create mode 100644 workbench_scripts/README.md diff --git a/workbench_scripts/README.md b/workbench_scripts/README.md new file mode 100644 index 0000000..08535fc --- /dev/null +++ b/workbench_scripts/README.md @@ -0,0 +1,60 @@ +# Cyclus - Workbench Integration + +You can download workbench from [the ORNL gitlab](https://code.ornl.gov/neams-workbench/downloads/-/tree/master). + +## Known bugs + +1. If you run on python3, and errors when you run `generate_cyclus_sch.py` on `xml2obj`, try with python 2. + +## Steps + +1. Go into `generate_cyclus_sch.py` and go to the very bottom, modify `path` to your workbench rte path. +2. Run `python generate_cyclus_sch.py`. This script will: + - Try to get the metadata from your Cyclus installation (Running `cyclus -m`) + - If this fails, it will just use the pre-shipped file (`m.json`) + - If you have it set up differently, change the `cyclus_cmd` variable. + - It will generate the following files into your respective workbench directories: + - Grammar file (`cyclus.wbg`) + - This file tells Workbench what files are needed + - and what syntax the input helper would use + - Schema file (`cyclus.sch`) + - schema file has all the rules for syntax validation + - and template pointers + - Template file + - Every archetype gets a template, so it's easier for users to know what attributes each has. + - It also comes with the docstring + - Highlight file + - This file defines different coloring (e.g. for comments, brackets etc.) + - Cyclus runner (`cyclus.py`) + - The runner is the part that `talks` to Workbench + - The runner has modules for taking in executables, + - running cyclus remotely / locally + - Converting files from SON to JSON + - Cyclus processor (`cyclus_processor.py`, `cyclus.wbp`) + - This is not used, due to some bugs. +3. Open Workbench, click on `file -> configurations`. +4. On the very top row, click `Add..` to add Cyclus +5. Configure your executable path, and if you're using a remote server to run your jobs, fill out the remote server address, username, and password +6. Create a new file, with extension `.cyclus` +7. Note that the text format will not be acceptable by Cyclus. `cyclus.py` will convert the SON file format to JSON readable for Cyclus. +8. If not already, click the dropdown next to `Processors` and choose Cyclus. +9. On the text editor, press `control + space` (for mac, other OSs look at `Edit -> Autocomplete` for shortcut) +10. click `simulation`, and the initial template will (hopefully) take it from there. +11. Notice the `validation` tab on the bottom, it will tell you if there's something wrong. +12. Note that if you have validation errors in the block you've been working on, Autocomplete won't work elsewhere. +13. Once finished, click `Run`. That will do: + - convert the current SON file you have to a JSON, and clean it up (outputs as `[your_filename].json`) + - if you defined a remote address: + - uploads the .json file into `/home/[username]/[some_hash]/input.json` + - runs cyclus + - downloads the output to `[your_filename].sqlite` + - if you didn't: + - runs cyclus by `[your_executable_command] [your_filename].json -o [your_filename].sqlite` + +## PS + +The postprocessor was written, and it does: +- Reads the .sqlite file and generates a long .csv file with 'important metrics' such as material flow +- Workbench can read those .csv files (but not .sqlite files, thus the conversion) and display them as plots and tables. + +It is currently commented out in `cyclus.py` (in function `postrun`), but feel free to play with it and see if it's worth it. I'd encourage eventual integration of Cymetric, obviously. \ No newline at end of file diff --git a/workbench_scripts/cyclus.py b/workbench_scripts/cyclus.py index 9359774..9a88ce9 100644 --- a/workbench_scripts/cyclus.py +++ b/workbench_scripts/cyclus.py @@ -15,6 +15,11 @@ import sys import tempfile import threading +here = os.path.dirname(os.path.abspath(__file__)) + +sys.path.append(os.path.join(here, 'cyclus')) +#from cyclus_processor import CyclusPostrunner + def unpack_stringlist(stringlist): """parses stringlist that was formatted to pass on command-line into original array of strings""" @@ -235,7 +240,7 @@ def __add_options(self): "type": "stringlist" }) shared.append({ - "default": self.executable, + "default": "cyclus", #self.executable, "dest": "executable", "flag": "-e", "help": "Path to the executable to run", @@ -530,6 +535,10 @@ def output_directory_overridden(self): return self.output_directory != None def postrun(self, options): + # something Cymetric, I suppose + # self.echo(1, '#### Postrunner on %s' %self.output_path) + # CyclusPostrunner(self.output_path) + # self.echo(1, '#### Finished Postrunner.') """actions to perform after the run finishes""" def prerun(self, options): @@ -547,6 +556,21 @@ def prerun(self, options): self.echo(2, "# ", options.working_directory) os.chdir(options.working_directory) + + # convert son to JSON and clean the JSON + binpath = os.path.join(here, os.pardir, 'bin') + sonjson_path = os.path.join(binpath, 'sonjson') + self.echo(1, '#### Converting SON to JSON.... ') + schema_file_path = os.path.join(here, 'cyclus', 'cyclus.sch') + p = subprocess.Popen([sonjson_path, schema_file_path, options.input], + stdout=subprocess.PIPE) + json_str = p.stdout.read() + extension = options.input.split('.')[-1] + self.json_filepath = options.input.replace(extension, 'json') + temp_json_path = os.path.join(self.working_directory, self.json_filepath) + with open(temp_json_path, 'w') as f: + f.write(self.clean_json(json_str)) + self.echo(1, '#### Finished converting to JSON! ') self.echo(1, "#### Pre-run ####") self.echo(1) @@ -590,17 +614,33 @@ def process_args(self, parser, args): self.echo(1, '# Connected.') self.echo(1, '# Testing if the executable exists...') + self.echo(1, '# Running "command -v %s"' %self.executable) # check if file is executable - output = self.remote_execute('test -x %s && echo "yayyy"' %self.executable) - if 'yay' in output: - self.echo(1, '# The file in the defined path is executable.') + output = self.remote_execute('command -v ' + self.executable) + if not output: + self.echo(0, '# The executable does not seem to exist...') + self.echo(0, '# Let me try something here and try to see if it is defined in bashrc...') + out = self.remote_execute('cat ~/.bashrc | grep "alias %s"' %self.executable) + if len(out) != 0: + self.echo(0, '# We found \n%s\n.. gonna try if this works' %out) + potential_cmd = out.strip().split('\n')[-1].split('=')[-1].strip() + out2 = self.remote_execute(potential_cmd) + if 'No input file' in out2: + self.echo(0, '# seems to have worked. Gonna Switch executable to %s' %potential_cmd) + self.executable = potential_cmd + else: + self.echo(0, '# Could not find executable path. Try the full path to the executable instead of an alias or command.') + sys.exit(1) + else: + self.echo(0, '# Could not find executable path. Try the full path to the executable instead of an alias or command.') + sys.exit(1) else: - self.echo(0, 'The file is not Executable') - sys.exit(1) + self.echo(1, 'The executable is good!') + except Exception as e: self.echo(0, 'Could not connect. Check arguments.') self.echo(0, 'See Error below:') - self.echo(0, e) + print(e) sys.exit(1) else: @@ -669,6 +709,8 @@ def process_args(self, parser, args): self.echo(1) def remote_execute(self, cmd): + self.echo(1, '## Remotely executing command:') + self.echo(1, cmd) i, o, e = self.ssh.exec_command(cmd) output = '\n'.join(o.readlines()) error = '\n'.join(e.readlines()) @@ -677,6 +719,33 @@ def remote_execute(self, cmd): return output + def clean_json(self, s): + news = [] + was_value = False + #was_null = False + for indx, line in enumerate(s.split('\n')): + if '"null"' in line: + line = line.replace('"null"', r'{}') + #if was_null: + # line = line.replace(']', '') + if was_value: + line = line.replace('}', '') + was_value = False + if '"value"' in line: + news[-1] = news[-1].replace('{', '') + news.append(line.replace('"value":', '')) + was_value = True + #if '"null"' in line: + # news[-1] = news[-1].replace('[', '') + # news.append(line.replace('"null"', r'{}')) + # was_null = True + else: + news.append(line) + parsed = json.loads(''.join(news)) + + return json.dumps(parsed, indent=4, sort_keys=True) + + def run(self, options): """run the given executable""" self.echo(1, "#### Run ", self.app_name(), " ####") @@ -686,7 +755,6 @@ def run(self, options): # Workbench's python environment as this is likely more # recent and contains packages that are not available # with default Python installations - print(self.executable) if self.executable.endswith(".py"): args = [sys.executable, self.executable] else: @@ -697,8 +765,7 @@ def run(self, options): args.extend(self.additional) # request list of supported arguments to pass to the executable args.extend(self.run_args(options)) - print('args') - print(args) + self.output_path = self.json_filepath.replace('.json', '.sqlite') if self.is_remote: self.echo(1, "#### Executing '", " ".join(args), "' on remote server %s " %self.remote_server_address) rtncode = 0 @@ -712,10 +779,9 @@ def run(self, options): import os while duplicate_hash and n < 3: rnd_dir = os.path.join('/home/', self.remote_server_username, str(uuid.uuid4())) - remote_input_path = os.path.join(rnd_dir, 'input.xml') - remote_output_path = remote_input_path.replace('.xml', '.sqlite') + remote_input_path = os.path.join(rnd_dir, 'input.json') + remote_output_path = remote_input_path.replace('.json', '.sqlite') output = self.remote_execute('mkdir %s' %rnd_dir) - print('error', output) n+=1 if not output: # empty output means nothing went wrong, @@ -723,12 +789,14 @@ def run(self, options): self.echo(1, '# Uploading input file to %s' %self.remote_server_address) self.echo(1, '# To path "%s"' %remote_input_path) ftp = self.ssh.open_sftp() - ftp.put(options.input ,remote_input_path) + ftp.put(self.json_filepath ,remote_input_path) self.echo(1, '# Now running %s...' %self.app_name()) - output = self.remote_execute('%s %s -o %s --warn-limit 0' %(self.executable, remote_input_path, remote_output_path)) + cmd = '%s %s -o %s --warn-limit 0' %(self.executable, remote_input_path, remote_output_path) + self.echo(1, '# Running command: %s' %cmd) + output = self.remote_execute(cmd) # this is super wonky, consider changing - if output == 0 or ('Error' not in output and 'error' not in output and 'Abort' not in output and 'fatal' not in output and 'Invalid' not in output): + if output == 0 or ('ERROR' not in output.upper() and 'ABORT' not in output.upper() and 'FATAL' not in output.upper() and 'INVALID' not in output.upper() and 'FAILED TO VALIDATE' not in output.upper()): self.echo(1, '############################' ) self.echo(1, '# %s ran successfully!' %self.app_name()) @@ -736,20 +804,21 @@ def run(self, options): self.echo(1, '# Now downloading output file') pre, ext = os.path.splitext(options.input) - ftp.get(remote_output_path, os.path.join(self.working_directory, pre + '.out')) + + self.pre = pre + ftp.get(remote_output_path, self.output_path) # this is super wonky, consider changing - time.sleep(5) - self.echo(1, '# Download complete (%s)' %os.path.join(self.working_directory, pre + '.out')) + time.sleep(10) + self.echo(1, '# Download complete (%s)' %self.output_path) else: self.echo(1, '# Run Failed! See the following output') self.echo(1, output) except Exception as e: - self.echo(0, 'Something Went Wrong') + self.echo(0, 'Something went wrong during running Cyclus remotely:') self.echo(0, 'See Error below:') print(e) - self.echo(0, e) sys.exit(1) else: @@ -758,24 +827,27 @@ def run(self, options): # execute rtncode = 0 try: - proc = subprocess.Popen(args, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) - - # tee-output objects - teeout = None - teeerr = None - - # tee requested - if self.tee: - teeout = open(options.output_basename + ".out", "w") - teeerr = open(options.output_basename + ".err", "w") - - # start background readers - out = threading.Thread(target=streamer, name="out_reader", - args=(proc.stdout, sys.stdout, teeout)) - err = threading.Thread(target=streamer, name="err_reader", - args=(proc.stderr, sys.stderr, teeerr)) - out.start() - err.start() + args = [self.executable, self.json_filepath, '-o', self.output_path] + proc = subprocess.Popen(args, bufsize=0, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + + if False: + # tee-output objects + teeout = None + teeerr = None + + # tee requested + if self.tee: + teeout = open(options.output_basename + ".out", "w") + teeerr = open(options.output_basename + ".err", "w") + + # start background readers + out = threading.Thread(target=streamer, name="out_reader", + args=(proc.stdout, sys.stdout, teeout)) + err = threading.Thread(target=streamer, name="err_reader", + args=(proc.stderr, sys.stderr, teeerr)) + out.start() + err.start() # wait for process to finish proc.wait() diff --git a/workbench_scripts/generate_cyclus_sch.py b/workbench_scripts/generate_cyclus_sch.py index 005bb88..9481ec9 100644 --- a/workbench_scripts/generate_cyclus_sch.py +++ b/workbench_scripts/generate_cyclus_sch.py @@ -1,6 +1,7 @@ #import xmltodict import copy import numpy as np +import shutil import json import re import os @@ -333,11 +334,14 @@ def get_cyclus_files(self): # this is where everything happens # temporary !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - #p = subprocess.Popen([self.cyclus_cmd, '-m'], stdout=subprocess.PIPE) - #meta_str = p.stdout.read() - #self.meta_dict = json.loads(meta_str) - heredir = os.path.abspath(os.path.dirname(__file__)) - self.meta_dict = json.loads(open(os.path.join(heredir, 'm.json')).read()) + try: + p = subprocess.Popen([self.cyclus_cmd, '-m'], stdout=subprocess.PIPE) + meta_str = p.stdout.read() + self.meta_dict = json.loads(meta_str) + except: + print('Could not run Cyclus, replacing metadata file with a pre-stored one..') + heredir = os.path.abspath(os.path.dirname(__file__)) + self.meta_dict = json.loads(open(os.path.join(heredir, 'm.json')).read()) # temporary !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @@ -351,8 +355,9 @@ def get_cyclus_files(self): name = arche.split(':')[-1] self.type_dict[name] = self.meta_dict['annotations'][arche]['entity'] self.schema_dict[name] = {'InputTmpl': '"%s"' %name.encode('ascii')} + spec_string += ' '*16 + 'spec = {lib="%s" name="%s"}\n' %(arche.split(':')[1], arche.split(':')[2]) if 'NullRegion' in arche or 'NullInst' in arche: - self.template_dict[name] = name+'= null' + self.template_dict[name] = name+r'={}' continue d = dict(xml2obj(self.meta_dict['schema'][arche])._attrs) #d = xmltodict.parse(self.meta_dict['schema'][arche])['interleave'] @@ -372,7 +377,6 @@ def get_cyclus_files(self): # fill in init_template to have the archetypes - spec_string += ' '*16 + 'spec = {lib="%s" name="%s"}\n' %(arche.split(':')[1], arche.split(':')[2]) self.init_template = self.init_template.replace('$$spec_string', spec_string) self.template_dict['init_template'] = self.init_template @@ -588,14 +592,22 @@ def read_interleave(self, intd, name, from_one_or_more, optional): return d - -def generate_cyclus_workbench_files(#schema_path='/Users/4ib/Desktop/git/cyclus_gui/neams/cyclus.sch', - schema_path='/Users/4ib/Desktop/git/cyclus_gui/neams/cyclus.sch', - template_dir='/Users/4ib/Desktop/git/cyclus_gui/neams/templates/', - highlight_path='/Users/4ib/Desktop/git/cyclus_gui/neams/cyclus.wbh', - grammar_path='/Users/4ib/.workbench/2.0.0/grammars/cyclus.wbg', - cyclus_cmd='cyclus'): +here = os.path.dirname(os.path.abspath(__file__)) +def generate_cyclus_workbench_files(workbench_rte_dir, + cyclus_cmd): # settings part + cyclus_dir = os.path.join(workbench_rte_dir, 'cyclus') + if not os.path.exists(cyclus_dir): + os.mkdir(cyclus_dir) + etc_dir = os.path.join(workbench_rte_dir, os.pardir, 'etc') + # define paths + schema_path = os.path.join(cyclus_dir, 'cyclus.sch') + template_dir = os.path.join(etc_dir, 'Templates', 'cyclus') + highlight_path = os.path.join(etc_dir, 'grammars', 'highlighters', 'cyclus.wbh') + grammar_path = os.path.join(etc_dir, 'grammars', 'cyclus.wbg') + + + grammar_str = """name= Cyclus enabled = true @@ -613,6 +625,8 @@ def generate_cyclus_workbench_files(#schema_path='/Users/4ib/Desktop/git/cyclus_ # write the files with open(grammar_path, 'w') as f: + print('Wrote grammar file at:') + print(grammar_path) f.write(grammar_str) # extra copy for giggles with open(schema_path.replace('.sch', '.wbg'), 'w') as f: @@ -620,17 +634,41 @@ def generate_cyclus_workbench_files(#schema_path='/Users/4ib/Desktop/git/cyclus_ s_ = generate_schema(cyclus_cmd) with open(schema_path, 'w') as f: + print('Wrote schema file at:') + print(schema_path) f.write(s_.sch_str) + # create template directory if it doesn't exist + if not os.path.exists(template_dir): + os.mkdir(template_dir) + print('Writing template files on %s:' %template_dir) for key, val in s_.template_dict.items(): with open(os.path.join(template_dir, key+'.tmpl'), 'w') as f: + print('\tWrote template for %s at:' %key) + print('\t'+os.path.join(template_dir, key+'.tmpl')) f.write(val) h_ = highlighter() with open(highlight_path, 'w') as f: + print('Wrote highlight file at:') + print(highlight_path) f.write(h_.highlight_str) - print('Done!') + + # copy the cyclus.py file + shutil.copyfile(os.path.join(here, 'cyclus.py'), os.path.join(workbench_rte_dir, 'cyclus.py')) + print('Copied cyclus runner to:') + print(os.path.join(workbench_rte_dir, 'cyclus.py')) + shutil.copyfile(os.path.join(here, 'cyclus_processor.py'), os.path.join(workbench_rte_dir, 'cyclus', 'cyclus_processor.py')) + print('Copied cyclus processor to:') + print(os.path.join(workbench_rte_dir, 'cyclus', 'cyclus_processor.py')) + shutil.copyfile(os.path.join(here, 'cyclus.wbp'), os.path.join(etc_dir, 'processors', 'cyclus.wbp')) + print('Copied cyclus processor file to:') + print(os.path.join(etc_dir, 'processors', 'cyclus.wbp')) + + print('Done!\n\n') + + def clean_xml(s): new = [] @@ -653,4 +691,8 @@ def clean_xml(s): if __name__ == '__main__': - generate_cyclus_workbench_files() \ No newline at end of file + # modify this for your setup! + path = '/Users/4ib/Downloads/Workbench-Darwin/rte' + cyclus_cmd = 'cyclus' + generate_cyclus_workbench_files(workbench_rte_dir=path, + cyclus_cmd=cyclus_cmd) \ No newline at end of file