[question] Monitoring a custom environment

I am training an A2C algorithm on a custom environment using multiprocessing and SubprocVecEnv as follows:

`env = SubprocVecEnv([lambda: CustomEnv(args, i) for i in range(args.cpus)])

model = A2C(MlpLnLstmPolicy, env, verbose=1, tensorboard_log=None, learning_rate=7e-3, lr_schedule="linear")

model.learn(total_timesteps=args.training_steps, log_interval=10)

I want to monitor the learning and save model checkpoints using a Monitor and callbacks, however I can’t seem to figure out how to combine everything. I’ve tried doing

`env = SubprocVecEnv([lambda: CustomEnv(args, i) for i in range(args.cpus)]) env = Monitor(env, log_dir, allow_early_resets=True)

model = A2C(MlpLnLstmPolicy, env, verbose=1, tensorboard_log=None, learning_rate=7e-3, lr_schedule="linear")

model.learn(total_timesteps=args.training_steps, log_interval=10, callback=callback)`

but I get the following exception:

Traceback (most recent call last): File “/main_file.py”, line 96, in <module> env = train(args) File “/main_file.py”, line 76, in train env = Monitor(env, log_dir, allow_early_resets=True) File “/miniconda3/envs/RL/lib/python3.7/site-packages/stable_baselines/bench/monitor.py”, line 27, in init Wrapper.init(self, env=env) File “/miniconda3/envs/RL/lib/python3.7/site-packages/gym/core.py”, line 210, in init self.reward_range = self.env.reward_range AttributeError: ‘SubprocVecEnv’ object has no attribute ‘reward_range’

So what is the correct way of using a monitor in this setting?

[ajtanskanen]

I also ran into similar problems with monitoring vectorized environments. It was straight forward to tweak VecMonitor from OpenAI Baselines to work with Stable Baselines, as suggested above. This is what I ended up with.

from stable_baselines.common.vec_env import VecEnvWrapper
import numpy as np
import time
from collections import deque
import os.path as osp
import json
import csv

class VecMonitor(VecEnvWrapper):
    EXT = "monitor.csv"
    
    def __init__(self, venv, filename=None, keep_buf=0, info_keywords=()):
        VecEnvWrapper.__init__(self, venv)
        print('init vecmonitor: ',filename)
        self.eprets = None
        self.eplens = None
        self.epcount = 0
        self.tstart = time.time()
        if filename:
            self.results_writer = ResultsWriter(filename, header={'t_start': self.tstart},
                extra_keys=info_keywords)
        else:
            self.results_writer = None
        self.info_keywords = info_keywords
        self.keep_buf = keep_buf
        if self.keep_buf:
            self.epret_buf = deque([], maxlen=keep_buf)
            self.eplen_buf = deque([], maxlen=keep_buf)

    def reset(self):
        obs = self.venv.reset()
        self.eprets = np.zeros(self.num_envs, 'f')
        self.eplens = np.zeros(self.num_envs, 'i')
        return obs

    def step_wait(self):
        obs, rews, dones, infos = self.venv.step_wait()
        self.eprets += rews
        self.eplens += 1

        newinfos = list(infos[:])
        for i in range(len(dones)):
            if dones[i]:
                info = infos[i].copy()
                ret = self.eprets[i]
                eplen = self.eplens[i]
                epinfo = {'r': ret, 'l': eplen, 't': round(time.time() - self.tstart, 6)}
                for k in self.info_keywords:
                    epinfo[k] = info[k]
                info['episode'] = epinfo
                if self.keep_buf:
                    self.epret_buf.append(ret)
                    self.eplen_buf.append(eplen)
                self.epcount += 1
                self.eprets[i] = 0
                self.eplens[i] = 0
                if self.results_writer:
                    self.results_writer.write_row(epinfo)
                newinfos[i] = info
        return obs, rews, dones, newinfos
        
class ResultsWriter(object):
    def __init__(self, filename, header='', extra_keys=()):
        print('init resultswriter')
        self.extra_keys = extra_keys
        assert filename is not None
        if not filename.endswith(VecMonitor.EXT):
            if osp.isdir(filename):
                filename = osp.join(filename, VecMonitor.EXT)
            else:
                filename = filename #   + "." + VecMonitor.EXT
        self.f = open(filename, "wt")
        if isinstance(header, dict):
            header = '# {} \n'.format(json.dumps(header))
        self.f.write(header)
        self.logger = csv.DictWriter(self.f, fieldnames=('r', 'l', 't')+tuple(extra_keys))
        self.logger.writeheader()
        self.f.flush()

    def write_row(self, epinfo):
        if self.logger:
            self.logger.writerow(epinfo)
            self.f.flush()        

[Demetrio92]

So in SB3 simply:

from stable_baselines3.common.vec_env import VecMonitor
env = VecMonitor(env, log_dir)

instead of

from stable_baselines3.common.monitor import Monitor
env = Monitor(env, log_dir)   # won't work with vectorized enviroments, will throw cryptic errors

---

sorry for off-topic, googled the error and got here