diff --git a/package/AUTHORS b/package/AUTHORS index ed82321a9e..9d7c6b9dbd 100644 --- a/package/AUTHORS +++ b/package/AUTHORS @@ -284,6 +284,7 @@ Chronological list of authors - Sai Udayagiri - Apoorva Verma - Aryaman Chaudhri + - Erik Fransson External code ------------- diff --git a/package/CHANGELOG b/package/CHANGELOG index 3e7c3d309d..99c7e1e3f4 100644 --- a/package/CHANGELOG +++ b/package/CHANGELOG @@ -18,11 +18,16 @@ The rules for this file: spyke7, talagayev, tanii1125, BradyAJohnston, hejamu, jeremyleung521, harshitgajjela-droid, kunjsinha, aygarwal, jauy123, Dreamstick9, ollyfutur, Amarendra22, charity-g, ParthUppal523, apoorva-01, RMeli, - raulloiscuns, Aryaman-Chaudhri + raulloiscuns, Aryaman-Chaudhri, erikfransson * 2.11.0 Fixes + * The NCDF reader now releases the pages of the memory map once a frame has + been read into the Timestep. Reading a netCDF trajectory with `mmap=True` + (the default for a file name) no longer grows the resident memory of the + process towards the size of the whole trajectory file. Requires a platform + that provides `MADV_DONTNEED`, i.e. not Windows (Discussion #4792). * `AtomGroup.rotate()` and the `rotateby` trajectory transformation now also rotate velocities and forces besides positions. This also affects `MDAnalysis.analysis.align.alignto()` and `AlignTraj`, since they diff --git a/package/MDAnalysis/coordinates/TRJ.py b/package/MDAnalysis/coordinates/TRJ.py index e9799e4d94..143f212640 100644 --- a/package/MDAnalysis/coordinates/TRJ.py +++ b/package/MDAnalysis/coordinates/TRJ.py @@ -134,6 +134,7 @@ import warnings import errno import logging +import mmap from math import isclose import MDAnalysis @@ -422,6 +423,11 @@ class NCDFReader(base.ReaderBase): the first two frames of the trajectory. :meth:`Writer` now also sets `convert_units`, `velocities`, `forces` and `scale_factor` information for the :class:`NCDFWriter`. + .. versionchanged:: 2.11.0 + The pages of the memory map are released after every frame, so that + memory use no longer grows towards the size of the trajectory file + while it is read. Requires ``MADV_DONTNEED``, which is not available + on Windows. """ @@ -675,6 +681,10 @@ def _read_frame(self, frame): self.convert_pos_from_native(ts.dimensions[:3]) ts.frame = frame # frame labels are 0-based self._current_frame = frame + # the frame is now copied into ts, so its pages can be dropped again + mm = getattr(self.trjfile, "_mm", None) + if mm is not None and hasattr(mmap, "MADV_DONTNEED"): + mm.madvise(mmap.MADV_DONTNEED) return ts def _reopen(self): diff --git a/testsuite/MDAnalysisTests/coordinates/test_netcdf.py b/testsuite/MDAnalysisTests/coordinates/test_netcdf.py index 1ebad11665..aedd45a38c 100644 --- a/testsuite/MDAnalysisTests/coordinates/test_netcdf.py +++ b/testsuite/MDAnalysisTests/coordinates/test_netcdf.py @@ -22,9 +22,11 @@ # import MDAnalysis as mda import numpy as np +import mmap import sys from scipy.io import netcdf_file +from types import SimpleNamespace import pytest from numpy.testing import assert_equal, assert_almost_equal @@ -148,6 +150,22 @@ class TestNCDFReaderTZ2(_NCDFReaderTest, RefTZ2): pass +@pytest.mark.skipif( + not hasattr(mmap, "MADV_DONTNEED"), reason="no MADV_DONTNEED" +) +def test_mmap_pages_dropped(monkeypatch): + """Reading a frame releases its pages of the memory map.""" + universe = mda.Universe(PRM_NCBOX, TRJ_NCBOX, mmap=True) + advice = [] + monkeypatch.setattr( + universe.trajectory.trjfile, + "_mm", + SimpleNamespace(madvise=advice.append), + ) + universe.trajectory[1] + assert advice == [mmap.MADV_DONTNEED] + + class TestNCDFReader2(object): """NCDF Trajectory with positions and forces.