Release netCDF memory-map pages after each frame

package/AUTHORS

@@ -284,6 +284,7 @@ Chronological list of authors
   - Sai Udayagiri
   - Apoorva Verma
   - Aryaman Chaudhri
+  - Erik Fransson
 
 External code
 -------------

package/CHANGELOG

@@ -18,11 +18,16 @@ The rules for this file:
          spyke7, talagayev, tanii1125, BradyAJohnston, hejamu, jeremyleung521,
          harshitgajjela-droid, kunjsinha, aygarwal, jauy123, Dreamstick9,
          ollyfutur, Amarendra22, charity-g, ParthUppal523, apoorva-01, RMeli,
-         raulloiscuns, Aryaman-Chaudhri
+         raulloiscuns, Aryaman-Chaudhri, erikfransson
 
  * 2.11.0
 
 Fixes
+ * The NCDF reader now releases the pages of the memory map once a frame has
+   been read into the Timestep. Reading a netCDF trajectory with `mmap=True`
+   (the default for a file name) no longer grows the resident memory of the
+   process towards the size of the whole trajectory file. Requires a platform
+   that provides `MADV_DONTNEED`, i.e. not Windows (Discussion #4792).
  * `AtomGroup.rotate()` and the `rotateby` trajectory transformation now
    also rotate velocities and forces besides positions. This also affects
    `MDAnalysis.analysis.align.alignto()` and `AlignTraj`, since they

package/MDAnalysis/coordinates/TRJ.py

@@ -134,6 +134,7 @@
 import warnings
 import errno
 import logging
+import mmap
 from math import isclose
 
 import MDAnalysis
@@ -422,6 +423,11 @@ class NCDFReader(base.ReaderBase):
        the first two frames of the trajectory.
        :meth:`Writer` now also sets `convert_units`, `velocities`, `forces` and
        `scale_factor` information for the :class:`NCDFWriter`.
+    .. versionchanged:: 2.11.0
+       The pages of the memory map are released after every frame, so that
+       memory use no longer grows towards the size of the trajectory file
+       while it is read. Requires ``MADV_DONTNEED``, which is not available
+       on Windows.
 
     """
 
@@ -675,6 +681,10 @@ def _read_frame(self, frame):
                 self.convert_pos_from_native(ts.dimensions[:3])
         ts.frame = frame  # frame labels are 0-based
         self._current_frame = frame
+        # the frame is now copied into ts, so its pages can be dropped again
+        mm = getattr(self.trjfile, "_mm", None)
+        if mm is not None and hasattr(mmap, "MADV_DONTNEED"):
+            mm.madvise(mmap.MADV_DONTNEED)
         return ts
 
     def _reopen(self):

testsuite/MDAnalysisTests/coordinates/test_netcdf.py

@@ -22,9 +22,11 @@
 #
 import MDAnalysis as mda
 import numpy as np
+import mmap
 import sys
 
 from scipy.io import netcdf_file
+from types import SimpleNamespace
 
 import pytest
 from numpy.testing import assert_equal, assert_almost_equal
@@ -148,6 +150,22 @@ class TestNCDFReaderTZ2(_NCDFReaderTest, RefTZ2):
     pass
 
 
+@pytest.mark.skipif(
+    not hasattr(mmap, "MADV_DONTNEED"), reason="no MADV_DONTNEED"
+)
+def test_mmap_pages_dropped(monkeypatch):
+    """Reading a frame releases its pages of the memory map."""
+    universe = mda.Universe(PRM_NCBOX, TRJ_NCBOX, mmap=True)
+    advice = []
+    monkeypatch.setattr(
+        universe.trajectory.trjfile,
+        "_mm",
+        SimpleNamespace(madvise=advice.append),
+    )
+    universe.trajectory[1]
+    assert advice == [mmap.MADV_DONTNEED]
+
+
 class TestNCDFReader2(object):
     """NCDF Trajectory with positions and forces.