Add an xlogfile to the generation of the ttyrecs.

An xlogfile is a logfile that gives details of the episodes that have
just passed (https://nethackwiki.com/wiki/Xlogfile).  These were
previously being generated in the HACKDIR (temporary var dir). By
setting the SCOREPREFIX and touching the file, you can can change the
path that the file (always ending 'xlogfile') is generated.

NB: There is a lock based on fnctl on the logfile. To avoid multiple
nles in different trying to write to the same file (with the same savedir)
we prepend the process number, like ttyrecs. For writing to same savedir
within same process, an id should be added to both ttyrec and xlogfile.
This can be done in future (along with relabelling ttyrec to ttyrec2).

The log file records all COMPLETED episodes (in the NetHack sense),
since it derives from the features used to calculate top ten. In
standard usage it should lead to one row per episode, with all ttyrecs
from the same process in the same file. You can then find the ttyrec
corresponding to each row (start counting at 0th row).

include/nleobs.h

@@ -85,6 +85,7 @@ typedef struct nle_settings {
      *  Path to NetHack's game files.
      */
     char hackdir[4096];
+    char scoreprefix[4096];
     char options[32768];
     char wizkit[4096];
     /*

nle/env/base.py

@@ -292,8 +292,11 @@ def __init__(
                 self.savedir, "nle.%i.%%i.ttyrec.bz2" % os.getpid()
             )
             ttyrec = self._ttyrec_pattern % 0
+            # Create an xlogfile with the same format of name.
+            scoreprefix = ttyrec.replace("0.ttyrec.bz2", "")
         else:
             ttyrec = None
+            scoreprefix = None
 
         self.nethack = nethack.Nethack(
             observation_keys=self._observation_keys,
@@ -302,6 +305,7 @@ def __init__(
             ttyrec=ttyrec,
             wizard=wizard,
             spawn_monsters=spawn_monsters,
+            scoreprefix=scoreprefix,
         )
         self._close_nethack = weakref.finalize(self, self.nethack.close)
 

nle/nethack/nethack.py

@@ -159,6 +159,7 @@ def __init__(
         wizard=False,
         hackdir=HACKDIR,
         spawn_monsters=True,
+        scoreprefix="",
     ):
         self._copy = copy
 
@@ -175,9 +176,15 @@ def __init__(
 
         # Symlink a nhdat.
         os.symlink(os.path.join(hackdir, "nhdat"), os.path.join(self._vardir, "nhdat"))
-        # Touch a few files.
-        for fn in ["perm", "record", "logfile", "xlogfile"]:
+
+        # Touch files, so lock_file() in files.c passes.
+        for fn in ["perm", "record", "logfile"]:
             os.close(os.open(os.path.join(self._vardir, fn), os.O_CREAT))
+        if scoreprefix:
+            os.close(os.open(scoreprefix + "xlogfile", os.O_CREAT))
+        else:
+            os.close(os.open(os.path.join(self._vardir, "xlogfile"), os.O_CREAT))
+
         os.mkdir(os.path.join(self._vardir, "save"))
 
         # An assortment of hacks:
@@ -209,7 +216,12 @@ def __init__(
             )
         else:
             self._pynethack = _pynethack.Nethack(
-                self.dlpath, ttyrec, self._vardir, self._nethackoptions, spawn_monsters
+                self.dlpath,
+                ttyrec,
+                self._vardir,
+                self._nethackoptions,
+                spawn_monsters,
+                scoreprefix,
             )
         self._ttyrec = ttyrec
 

nle/scripts/play.py

@@ -86,6 +86,7 @@ def play():
     else:
         env = gym.make(
             FLAGS.env,
+            save_ttyrec_every=2,
             savedir=FLAGS.savedir,
             max_episode_steps=FLAGS.max_steps,
             allow_all_yn_questions=True,

nle/tests/test_envs.py

@@ -225,6 +225,9 @@ def test_rollout(self, env_name, rollout_len):
             assert os.path.exists(
                 os.path.join(savedir, "nle.%i.0.ttyrec.bz2" % os.getpid())
             )
+            assert os.path.exists(
+                os.path.join(savedir, "nle.%i.xlogfile" % os.getpid())
+            )
 
     def test_rollout_no_archive(self, env_name, rollout_len):
         """Tests rollout_len steps (or until termination) of random policy."""
@@ -319,7 +322,7 @@ def test_render_ansi(self, env_name, rollout_len):
 class TestGymDynamics:
     """Tests a few game dynamics."""
 
-    @pytest.fixture(autouse=True)  # will be applied to all tests in class
+    @pytest.fixture(autouse=True)  # Will be applied to all tests in class.
     def make_cwd_tmp(self, tmpdir):
         """Makes cwd point to the test's tmpdir."""
         with tmpdir.as_cwd():
@@ -373,14 +376,26 @@ def test_final_reward(self, env):
 
     def test_ttyrec_every(self):
         path = pathlib.Path(".")
-        env = gym.make("NetHackScore-v0", save_ttyrec_every=2, savedir=str(path))
+        env = gym.make("NetHackChallenge-v0", save_ttyrec_every=2, savedir=str(path))
+        pid = os.getpid()
         for episode in range(10):
             env.reset()
+            for c in [ord(" "), ord(" "), ord("<"), ord("y")]:
+                _, _, done, *_ = env.step(env.actions.index(c))
+            assert done
+
             if episode % 2 != 0:
                 continue
             contents = set(str(p) for p in path.iterdir())
-            assert len(contents) == episode // 2 + 1
-            assert "nle.%i.%i.ttyrec.bz2" % (os.getpid(), episode) in contents
+            # `contents` includes xlogfile and ttyrecs.
+            assert len(contents) - 1 == episode // 2 + 1
+            assert "nle.%i.%i.ttyrec.bz2" % (pid, episode) in contents
+            assert "nle.%i.xlogfile" % pid in contents
+
+        with open("nle.%i.xlogfile" % pid, "r") as f:
+            entries = f.readlines()
+
+        assert len(entries) == 10
 
 
 class TestEnvMisc:

src/nle.c

@@ -176,13 +176,16 @@ mainloop(fcontext_transfer_t ctx_transfer)
         settings.hackdir[len] = '\0';
     }
 
+    char *scoreprefix = (settings.scoreprefix[0] != '\0')
+                            ? settings.scoreprefix
+                            : settings.hackdir;
     fqn_prefix[SYSCONFPREFIX] = settings.hackdir;
     fqn_prefix[CONFIGPREFIX] = settings.hackdir;
     fqn_prefix[HACKPREFIX] = settings.hackdir;
     fqn_prefix[SAVEPREFIX] = settings.hackdir;
     fqn_prefix[LEVELPREFIX] = settings.hackdir;
     fqn_prefix[BONESPREFIX] = settings.hackdir;
-    fqn_prefix[SCOREPREFIX] = settings.hackdir;
+    fqn_prefix[SCOREPREFIX] = scoreprefix;
     fqn_prefix[LOCKPREFIX] = settings.hackdir;
     fqn_prefix[TROUBLEPREFIX] = settings.hackdir;
     fqn_prefix[DATAPREFIX] = settings.hackdir;

win/rl/pynethack.cc

@@ -91,7 +91,8 @@ class Nethack
 {
   public:
     Nethack(std::string dlpath, std::string ttyrec, std::string hackdir,
-            std::string nethackoptions, bool spawn_monsters)
+            std::string nethackoptions, bool spawn_monsters,
+            std::string scoreprefix)
         : Nethack(std::move(dlpath), std::move(hackdir),
                   std::move(nethackoptions), spawn_monsters)
     {
@@ -100,6 +101,16 @@ class Nethack
             PyErr_SetFromErrnoWithFilename(PyExc_OSError, ttyrec.c_str());
             throw py::error_already_set();
         }
+
+        if (ttyrec.size() > sizeof(settings_.scoreprefix) - 1) {
+            throw std::length_error("ttyrec filepath too long");
+        }
+
+        if (scoreprefix.size() > sizeof(settings_.scoreprefix) - 1) {
+            throw std::length_error("scoreprefix too long");
+        }
+        strncpy(settings_.scoreprefix, scoreprefix.c_str(),
+                scoreprefix.length());
     }
 
     Nethack(std::string dlpath, std::string hackdir,
@@ -338,9 +349,10 @@ PYBIND11_MODULE(_pynethack, m)
 
     py::class_<Nethack>(m, "Nethack")
         .def(py::init<std::string, std::string, std::string, std::string,
-                      bool>(),
+                      bool, std::string>(),
              py::arg("dlpath"), py::arg("ttyrec"), py::arg("hackdir"),
-             py::arg("nethackoptions"), py::arg("spawn_monsters") = true)
+             py::arg("nethackoptions"), py::arg("spawn_monsters") = true,
+             py::arg("scoreprefix") = "")
         .def(py::init<std::string, std::string, std::string, bool>(),
              py::arg("dlpath"), py::arg("hackdir"), py::arg("nethackoptions"),
              py::arg("spawn_monsters") = true)