# # Copyright (c) 2019 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # """Manage all command arguments.""" import os import signal import time import pandas as pd import numpy as np import pytest from rl_coach.tests.utils.test_utils import get_csv_path, get_files_from_dir, \ find_string_in_logs from rl_coach.tests.utils.definitions import Definitions as Def def collect_preset_for_mxnet(): """ Collect presets that relevant for args testing only. This used for testing arguments for specific presets that defined in the definitions (args_test under Presets). :return: preset(s) list """ for pn in Def.Presets.mxnet_args_test: assert pn, Def.Consts.ASSERT_MSG.format("Preset name", pn) yield pn def collect_preset_for_args(): """ Collect presets that relevant for args testing only. This used for testing arguments for specific presets that defined in the definitions (args_test under Presets). :return: preset(s) list """ for pn in Def.Presets.args_test: assert pn, Def.Consts.ASSERT_MSG.format("Preset name", pn) yield pn def collect_preset_for_seed(): """ Collect presets that relevant for seed argument testing only. This used for testing arguments for specific presets that defined in the definitions (args_test under Presets). :return: preset(s) list """ for pn in Def.Presets.args_for_seed_test: assert pn, Def.Consts.ASSERT_MSG.format("Preset name", pn) yield pn def collect_args(): """ Collect args from the cmd args list - on each test iteration, it will yield one line (one arg). :yield: one arg foe each test iteration """ for i in Def.Flags.cmd_args: assert i, Def.Consts.ASSERT_MSG.format("flag list", str(i)) yield i def add_one_flag_value(flag): """ Add value to flag format in order to run the python command with arguments. :param flag: dict flag :return: flag with format """ if not flag or len(flag) == 0: return [] if len(flag) == 1: return flag if Def.Flags.enw in flag[1]: flag[1] = '2' elif Def.Flags.css in flag[1]: flag[1] = '5' elif Def.Flags.fw_ten in flag[1]: flag[1] = "tensorflow" elif Def.Flags.fw_mx in flag[1]: flag[1] = "mxnet" elif Def.Flags.cp in flag[1]: flag[1] = "heatup_steps=EnvironmentSteps({})".format(Def.Consts.num_hs) return flag def is_reward_reached(csv_path, p_valid_params, start_time, time_limit): """ Check the result of the experiment, by collecting all the Evaluation Reward and average should be bigger than the min reward threshold. :param csv_path: csv file (results) :param p_valid_params: experiment test params :param start_time: start time of the test :param time_limit: timeout of the test :return: |Bool| true if reached the reward minimum """ win_size = 10 last_num_episodes = 0 csv = None reward_reached = False while csv is None or (csv['Episode #'].values[-1] < p_valid_params.max_episodes_to_achieve_reward and time.time() - start_time < time_limit): csv = pd.read_csv(csv_path) if 'Evaluation Reward' not in csv.keys(): continue rewards = csv['Evaluation Reward'].values rewards = rewards[~np.isnan(rewards)] if len(rewards) >= 1: averaged_rewards = np.convolve(rewards, np.ones( min(len(rewards), win_size)) / win_size, mode='valid') else: # May be in heat-up steps time.sleep(1) continue if csv['Episode #'].shape[0] - last_num_episodes <= 0: continue last_num_episodes = csv['Episode #'].values[-1] # check if reward is enough if np.any(averaged_rewards >= p_valid_params.min_reward_threshold): reward_reached = True break time.sleep(1) return reward_reached def validate_arg_result(flag, p_valid_params, clres=None, process=None, start_time=None, timeout=Def.TimeOuts.test_time_limit): """ Validate results of one argument. :param flag: flag to check :param p_valid_params: params test per preset :param clres: object of files paths (results of test experiment) :param process: process object :param start_time: start time of the test :param timeout: timeout of the test- fail test once over the timeout """ if flag[0] == "-ns" or flag[0] == "--no-summary": """ --no-summary: Once selected, summary lines shouldn't appear in logs """ # send CTRL+C to close experiment process.send_signal(signal.SIGINT) assert not find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.RESULTS_SORTED), \ Def.Consts.ASSERT_MSG.format("No Result summary", Def.Consts.RESULTS_SORTED) assert not find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.TOTAL_RUNTIME), \ Def.Consts.ASSERT_MSG.format("No Total runtime summary", Def.Consts.TOTAL_RUNTIME) assert not find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.DISCARD_EXP), \ Def.Consts.ASSERT_MSG.format("No discard message", Def.Consts.DISCARD_EXP) elif flag[0] == "-asc" or flag[0] == "--apply_stop_condition": """ -asc, --apply_stop_condition: Once selected, coach stopped when required success rate reached """ assert find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.REACHED_REQ_ASC, wait_and_find=True), \ Def.Consts.ASSERT_MSG.format(Def.Consts.REACHED_REQ_ASC, "Message Not Found") elif flag[0] == "--print_networks_summary": """ --print_networks_summary: Once selected, agent summary should appear in stdout. """ if find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.INPUT_EMBEDDER): assert True, Def.Consts.ASSERT_MSG.format( Def.Consts.INPUT_EMBEDDER, "Not found") if find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.MIDDLEWARE): assert True, Def.Consts.ASSERT_MSG.format( Def.Consts.MIDDLEWARE, "Not found") if find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.OUTPUT_HEAD): assert True, Def.Consts.ASSERT_MSG.format( Def.Consts.OUTPUT_HEAD, "Not found") elif flag[0] == "-tb" or flag[0] == "--tensorboard": """ -tb, --tensorboard: Once selected, a new folder should be created in experiment folder. """ csv_path = get_csv_path(clres) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) exp_path = os.path.dirname(csv_path[0]) tensorboard_path = os.path.join(exp_path, Def.Path.tensorboard) assert os.path.isdir(tensorboard_path), \ Def.Consts.ASSERT_MSG.format("tensorboard path", tensorboard_path) # check if folder contain files and check extensions files = get_files_from_dir(dir_path=tensorboard_path) assert any(".tfevents." in file for file in files) elif flag[0] == "-onnx" or flag[0] == "--export_onnx_graph": """ -onnx, --export_onnx_graph: Once selected, warning message should appear, it should be with another flag. """ assert find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.ONNX_WARNING, wait_and_find=True), \ Def.Consts.ASSERT_MSG.format(Def.Consts.ONNX_WARNING, "Not found") elif flag[0] == "-dg" or flag[0] == "--dump_gifs": """ -dg, --dump_gifs: Once selected, a new folder should be created in experiment folder for gifs files. """ pytest.xfail(reason="GUI issue on CI") csv_path = get_csv_path(clres) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) exp_path = os.path.dirname(csv_path[0]) gifs_path = os.path.join(exp_path, Def.Path.gifs) # wait until gif folder were created while time.time() - start_time < timeout: if os.path.isdir(gifs_path): assert os.path.isdir(gifs_path), \ Def.Consts.ASSERT_MSG.format("gifs path", gifs_path) break # check if folder contain files get_files_from_dir(dir_path=gifs_path) elif flag[0] == "-dm" or flag[0] == "--dump_mp4": """ -dm, --dump_mp4: Once selected, a new folder should be created in experiment folder for videos files. """ pytest.xfail(reason="GUI issue on CI") csv_path = get_csv_path(clres) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) exp_path = os.path.dirname(csv_path[0]) videos_path = os.path.join(exp_path, Def.Path.videos) # wait until video folder were created while time.time() - start_time < timeout: if os.path.isdir(videos_path): assert os.path.isdir(videos_path), \ Def.Consts.ASSERT_MSG.format("videos path", videos_path) break # check if folder contain files get_files_from_dir(dir_path=videos_path) elif flag[0] == "--nocolor": """ --nocolor: Once selected, check if color prefix is replacing the actual color; example: '## agent: ...' """ assert find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.COLOR_PREFIX, wait_and_find=True), \ Def.Consts.ASSERT_MSG.format(Def.Consts.COLOR_PREFIX, "Color Prefix Not Found") elif flag[0] == "--evaluate": """ --evaluate: Once selected, Coach start testing, there is not training. """ # wait until files created get_csv_path(clres=clres) time.sleep(15) assert not find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.TRAINING), \ Def.Consts.ASSERT_MSG.format("Training Not Found", Def.Consts.TRAINING) elif flag[0] == "--play": """ --play: Once selected alone, an warning message should appear, it should be with another flag. """ assert find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.PLAY_WARNING, wait_and_find=True), \ Def.Consts.ASSERT_MSG.format(Def.Consts.PLAY_WARNING, "Not found") elif flag[0] == "-et" or flag[0] == "--environment_type": """ -et, --environment_type: Once selected check csv results is created. """ csv_path = get_csv_path(clres) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) elif flag[0] == "-s" or flag[0] == "--checkpoint_save_secs": """ -s, --checkpoint_save_secs: Once selected, check if files added to the experiment path. """ csv_path = get_csv_path(clres) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) exp_path = os.path.dirname(csv_path[0]) checkpoint_path = os.path.join(exp_path, Def.Path.checkpoint) # wait until video folder were created while time.time() - start_time < timeout: if os.path.isdir(checkpoint_path): assert os.path.isdir(checkpoint_path), \ Def.Consts.ASSERT_MSG.format("checkpoint path", checkpoint_path) break # check if folder contain files get_files_from_dir(dir_path=checkpoint_path) elif flag[0] == "-ew" or flag[0] == "--evaluation_worker": """ -ew, --evaluation_worker: Once selected, check that an evaluation worker is created. e.g. by checking that it's csv file is created. """ # wait until files created csv_path = get_csv_path(clres=clres, extra_tries=10) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) elif flag[0] == "-cp" or flag[0] == "--custom_parameter": """ -cp, --custom_parameter: Once selected, check that the total steps are around the given param with +/- gap. also, check the heat-up param """ # wait until files created csv_path = get_csv_path(clres=clres) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) # read csv file csv = pd.read_csv(csv_path[0]) # check heat-up value results = [] if csv["In Heatup"].values[-1] == 0: results.append(csv["Total steps"].values[-1]) else: while csv["In Heatup"].values[-1] == 1: csv = pd.read_csv(csv_path[0]) last_step = csv["Total steps"].values results.append(last_step[-1]) time.sleep(1) assert int(results[-1]) >= Def.Consts.num_hs, \ Def.Consts.ASSERT_MSG.format("bigger than " + str(Def.Consts.num_hs), results[-1]) elif flag[0] == "-f" or flag[0] == "--framework": """ -f, --framework: Once selected, f = tensorflow or mxnet """ # wait until files created csv_path = get_csv_path(clres=clres) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) get_reward = is_reward_reached(csv_path=csv_path[0], p_valid_params=p_valid_params, start_time=start_time, time_limit=timeout) # check if experiment is working and reached the reward assert get_reward, Def.Consts.ASSERT_MSG.format( "Doesn't reached the reward", get_reward) # check if there is no exception assert not find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.LOG_ERROR) ret_val = process.poll() assert ret_val is None, Def.Consts.ASSERT_MSG.format("None", ret_val) elif flag[0] == "-crd" or flag[0] == "--checkpoint_restore_dir": """ -crd, --checkpoint_restore_dir: Once selected alone, check that can't restore checkpoint dir (negative test). """ # wait until files created csv_path = get_csv_path(clres=clres) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("path not found", csv_path) assert find_string_in_logs(log_path=clres.stdout.name, str=Def.Consts.NO_CHECKPOINT), \ Def.Consts.ASSERT_MSG.format(Def.Consts.NO_CHECKPOINT, "Not found") elif flag[0] == "--seed": """ --seed: Once selected, check logs of process list if all are the same results. """ lst_csv = [] # wait until files created csv_path = get_csv_path(clres=clres, extra_tries=20, num_expected_files=int(flag[1])) assert len(csv_path) > 0, \ Def.Consts.ASSERT_MSG.format("paths are not found", csv_path) assert int(flag[1]) == len(csv_path), Def.Consts.ASSERT_MSG. \ format(len(csv_path), int(flag[1])) # wait for getting results in csv's for i in range(len(csv_path)): lines_in_file = pd.read_csv(csv_path[i]) while len(lines_in_file['Episode #'].values) < 100 and \ time.time() - start_time < Def.TimeOuts.test_time_limit: lines_in_file = pd.read_csv(csv_path[i]) time.sleep(1) lst_csv.append(pd.read_csv(csv_path[i], nrows=Def.Consts.N_csv_lines)) assert len(lst_csv) > 1, Def.Consts.ASSERT_MSG.format("> 1", len(lst_csv)) df1 = lst_csv[0] for df in lst_csv[1:]: assert list(df1['Training Iter'].values) == list( df['Training Iter'].values) assert list(df1['ER #Transitions'].values) == list( df['ER #Transitions'].values) assert list(df1['Total steps'].values) == list( df['Total steps'].values) elif flag[0] == "-c" or flag[0] == "--use_cpu": pass elif flag[0] == "-n" or flag[0] == "--num_workers": """ -n, --num_workers: Once selected alone, check that csv created for each worker, and check results. """ # wait until files created csv_path = get_csv_path(clres=clres, extra_tries=20) num_expected_files = int(flag[1]) assert len(csv_path) >= num_expected_files, \ Def.Consts.ASSERT_MSG.format(str(num_expected_files), str(len(csv_path)))