Skip to content

Commit

Permalink
feat: add a view script
Browse files Browse the repository at this point in the history
  • Loading branch information
@xionghuichen
xionghuichen committed Apr 25, 2022
1 parent 5764414 commit f6fbc92
Show file tree
Hide file tree
Showing 3 changed files with 107 additions and 43 deletions.
122 changes: 81 additions & 41 deletions RLA/easy_log/log_tools.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,39 +28,13 @@ def __init__(self, optional_log_type=None):
if optional_log_type is not None:
self.log_types.extend(optional_log_type)

class DownloadLogTool(BasicLogTool):
def __init__(self, rla_config_path, proj_root, task, regex, *args, **kwargs):
fs = open(rla_config_path, encoding="UTF-8")
self.private_config = yaml.load(fs)
self.proj_root = proj_root
self.task_table_name = task
self.regex = regex

def _download_log(self, show=False):
for log_type in self.log_types:
root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex)
empty = True
for root_dir in glob.glob(root_dir_regex):

pass

class DeleteLogTool(BasicLogTool):
def __init__(self, proj_root, task_table_name, regex, filter, *args, **kwargs):
self.proj_root = proj_root
self.task_table_name = task_table_name
self.regex = regex
assert isinstance(filter, Filter)
self.filter = filter
self.small_timestep_regs = []
super(DeleteLogTool, self).__init__(*args, **kwargs)

def _find_small_timestep_log(self):
root_dir_regex = osp.join(self.proj_root, LOG, self.task_table_name, self.regex)
def _find_small_timestep_log(self, proj_root, task_table_name, regex, timstep_upper_bound=np.inf, timestep_lower_bound=0):
small_timestep_regs = []
root_dir_regex = osp.join(proj_root, LOG, task_table_name, regex)
for root_dir in glob.glob(root_dir_regex):
print("searching dirs", root_dir)
if os.path.exists(root_dir):
for file_list in os.walk(root_dir):

if re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]):
target_reg = re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]).group(0)
else:
Expand All @@ -77,7 +51,8 @@ def _find_small_timestep_log(self):
if file_list[1] == ['tb'] or os.path.exists(progress_csv_file): # in root of logdir
if not os.path.exists(progress_csv_file) or os.path.getsize(progress_csv_file) == 0:
print("[delete] find an experiment without progress.csv.", file_list[0])
self.small_timestep_regs.append(target_reg)
if timestep_lower_bound <= 0:
small_timestep_regs.append([target_reg, file_list[0]])
else:
try:
reader = pd.read_csv(progress_csv_file, chunksize=100000, quoting=csv.QUOTE_NONE,
Expand All @@ -88,9 +63,9 @@ def _find_small_timestep_log(self):
raw_df = pd.concat([raw_df, slim_chunk], ignore_index=True)
last_timestep = raw_df[DEFAULT_X_NAME].max()
print("[found a log] time_step ", last_timestep, target_reg)
if last_timestep < self.filter.timstep_bound:
self.small_timestep_regs.append(target_reg)
print("[delete] find an experiment with too small number of logs. ", file_list[0])
if timestep_lower_bound <= last_timestep <= timstep_upper_bound:
small_timestep_regs.append([target_reg, file_list[0]])
print("[delete] find an experiment satisfied timestep range. ", file_list[0])
else:
print("[valid]")
except Exception as e:
Expand All @@ -101,8 +76,35 @@ def _find_small_timestep_log(self):
elif 'events' in file_list[0]: # in event dir
pass
else: # empty dir
self.small_timestep_regs.append(target_reg)
if timestep_lower_bound <= 0:
small_timestep_regs.append([target_reg, file_list[0]])
print("[delete] find an experiment without any files. ", file_list[0])
return small_timestep_regs

class DownloadLogTool(BasicLogTool):
def __init__(self, rla_config_path, proj_root, task, regex, *args, **kwargs):
fs = open(rla_config_path, encoding="UTF-8")
self.private_config = yaml.load(fs)
self.proj_root = proj_root
self.task_table_name = task
self.regex = regex

def _download_log(self, show=False):
for log_type in self.log_types:
root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex)
empty = True
for root_dir in glob.glob(root_dir_regex):
pass

class DeleteLogTool(BasicLogTool):
def __init__(self, proj_root, task_table_name, regex, filter, *args, **kwargs):
self.proj_root = proj_root
self.task_table_name = task_table_name
self.regex = regex
assert isinstance(filter, Filter)
self.filter = filter
self.small_timestep_regs = []
super(DeleteLogTool, self).__init__(*args, **kwargs)

def _delete_related_log(self, regex, show=False):
log_found = 0
Expand Down Expand Up @@ -155,7 +157,7 @@ def delete_related_log(self, skip_ask=False):
return 0

def delete_small_timestep_log(self, skip_ask=False):
self._find_small_timestep_log()
self.small_timestep_regs = self._find_small_timestep_log(self.proj_root, self.task_table_name, self.regex, timstep_upper_bound=self.filter.timstep_bound)
print("complete searching.")
if skip_ask:
s = 'y'
Expand All @@ -164,17 +166,17 @@ def delete_small_timestep_log(self, skip_ask=False):
log_found = 0

if s == 'y' or skip_ask:
for reg in self.small_timestep_regs:
print("[delete small-timestep log] reg: ", reg)
self._delete_related_log(show=True, regex=reg + '*')
for res in self.small_timestep_regs:
print("[delete small-timestep log] reg: ", res[1])
self._delete_related_log(show=True, regex=res[0] + '*')
if skip_ask:
s = 'y'
else:
s = input("delete these files? (y/n)")
if s == 'y' or skip_ask:
for reg in self.small_timestep_regs:
print("do delete: ", reg)
log_found += self._delete_related_log(show=False, regex=reg + '*')
for res in self.small_timestep_regs:
print("do delete: ", res[1])
log_found += self._delete_related_log(show=False, regex=res[0] + '*')
return log_found

class ArchiveLogTool(BasicLogTool):
Expand Down Expand Up @@ -231,6 +233,44 @@ def archive_log(self, skip_ask=False):
print("do archive ...")
self._archive_log(show=False)

class ViewLogTool(BasicLogTool):
def __init__(self, proj_root, task_table_name, regex, *args, **kwargs):
self.proj_root = proj_root
self.task_table_name = task_table_name
self.regex = regex
super(ViewLogTool, self).__init__(*args, **kwargs)

def _view_log(self, regex):
root_dir_regex = osp.join(self.proj_root, LOG, self.task_table_name, regex)
for root_dir in glob.glob(root_dir_regex):
if os.path.exists(root_dir):
for file_list in os.walk(root_dir):
if re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]):
target_reg = re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]).group(0)
else:
target_reg = None
if target_reg is not None:
backup_file = file_list[0] + '/backup.txt'
if file_list[1] == ['tb'] or os.path.exists(backup_file): # in root of logdir
with open(backup_file) as f:
print(f.read())

def view_log(self, skip_ask=False):
found_regs = self._find_small_timestep_log(self.proj_root, self.task_table_name, self.regex, timestep_lower_bound=1)
for res in found_regs:
print("view experiments:", res[1])
if skip_ask:
s = 'y'
else:
s = input("press y to view \n ")
if s == 'y':
self._view_log(regex=res[0] + '*')






# if __name__ == '__main__':
# dlt = DeleteLogTool("../", "var_seq_imitation", "self-transfer", "2019/11/29/01-11*")
# dlt.delete_related_log()
19 changes: 19 additions & 0 deletions rla_scripts/view_expt.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
"""
A script to view data of experiments.
"""

from RLA.easy_log.log_tools import ViewLogTool
import argparse
from config import *

def argsparser():
parser = argparse.ArgumentParser("Archive Log")
parser.add_argument('--task_table_name', type=str)
parser.add_argument('--regex', type=str)
args = parser.parse_args()
return args

if __name__=='__main__':
args = argsparser()
dlt = ViewLogTool(proj_root=DATA_ROOT, task_table_name=args.task_table_name, regex=args.regex)
dlt.view_log()
9 changes: 7 additions & 2 deletions test/test_scripts.py
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from test._base import BaseTest
from RLA.easy_log.log_tools import DeleteLogTool, Filter
from RLA.easy_log.log_tools import ArchiveLogTool
from RLA.easy_log.log_tools import ArchiveLogTool, ViewLogTool


class ScriptTest(BaseTest):

Expand Down Expand Up @@ -51,4 +52,8 @@ def test_archive(self):
dlt = DeleteLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name='archived', regex='2022/03/01/21-13*', filter=filter)
log_found = dlt.delete_related_log(skip_ask=True)
assert log_found == 10


def test_view(self):
self.remove_and_copy_data()
dlt = ViewLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name=self.TASK_NAME, regex='2022/03/01/21-13*')
dlt.view_log(skip_ask=True)

0 comments on commit f6fbc92

Please sign in to comment.