diff --git a/.gitignore b/.gitignore index d1496d8..54a6ba8 100644 --- a/.gitignore +++ b/.gitignore @@ -151,3 +151,4 @@ /dnf-4.11.0.tar.gz /dnf-4.11.1.tar.gz /dnf-4.12.0.tar.gz +/dnf-4.13.0.tar.gz diff --git a/0001-Base.reset-plug-temporary-leak-of-libsolv-s-page-fil.patch b/0001-Base.reset-plug-temporary-leak-of-libsolv-s-page-fil.patch deleted file mode 100644 index 2162232..0000000 --- a/0001-Base.reset-plug-temporary-leak-of-libsolv-s-page-fil.patch +++ /dev/null @@ -1,317 +0,0 @@ -From 5ce5ed1ea08ad6e198c1c1642c4d9ea2db6eab86 Mon Sep 17 00:00:00 2001 -From: Laszlo Ersek -Date: Sun, 24 Apr 2022 09:08:28 +0200 -Subject: [PATCH] Base.reset: plug (temporary) leak of libsolv's page file - descriptors - -Consider the following call paths (mixed Python and C), extending from -livecd-creator down to libsolv: - - main [livecd-tools/tools/livecd-creator] - install() [livecd-tools/imgcreate/creator.py] - fill_sack() [dnf/dnf/base.py] - _add_repo_to_sack() [dnf/dnf/base.py] - load_repo() [libdnf/python/hawkey/sack-py.cpp] - dnf_sack_load_repo() [libdnf/libdnf/dnf-sack.cpp] - write_main() [libdnf/libdnf/dnf-sack.cpp] - repo_add_solv() [libsolv/src/repo_solv.c] - repopagestore_read_or_setup_pages() [libsolv/src/repopage.c] - dup() - write_ext() [libdnf/libdnf/dnf-sack.cpp] - repo_add_solv() [libsolv/src/repo_solv.c] - repopagestore_read_or_setup_pages() [libsolv/src/repopage.c] - dup() - -The dup() calls create the following file descriptors (output from -"lsof"): - -> COMMAND PID USER FD TYPE DEVICE SIZE/OFF NODE NAME -> python3 6500 root 7r REG 8,1 25320727 395438 /var/tmp/imgcreate-mytcghah/install_root/var/cache/dnf/fedora.solv (deleted) -> python3 6500 root 8r REG 8,1 52531426 395450 /var/tmp/imgcreate-mytcghah/install_root/var/cache/dnf/fedora-filenames.solvx - -These file descriptors are *owned* by the DnfSack object (which is derived -from GObject), as follows: - - sack->priv->pool->repos[1]->repodata[1]->store.pagefd = 7 - sack->priv->pool->repos[1]->repodata[2]->store.pagefd = 8 - ^ ^ ^ ^ ^ ^ ^ - | | | | | | | - | | | | | | int - | | | | | Repopagestore [libsolv/src/repopage.h] - | | | | Repodata [libsolv/src/repodata.h] - | | | struct s_Repo [libsolv/src/repo.h] - | | struct s_Pool (aka Pool) [libsolv/src/pool.h] - | DnfSackPrivate [libdnf/libdnf/dnf-sack.cpp] - DnfSack [libdnf/libdnf/dnf-sack.h] - -The file descriptors are *supposed* to be closed on the following call -path: - - main [livecd-tools/tools/livecd-creator] - install() [livecd-tools/imgcreate/creator.py] - close() [livecd-tools/imgcreate/dnfinst.py] - close() [dnf/dnf/base.py] - reset() [dnf/dnf/base.py] - _sack = None - _goal = None - _transaction = None - ... - dnf_sack_finalize() [libdnf/libdnf/dnf-sack.cpp] - pool_free() [libsolv/src/pool.c] - pool_freeallrepos() [libsolv/src/pool.c] - repo_freedata() [libsolv/src/repo.c] - repodata_freedata() [libsolv/src/repodata.c] - repopagestore_free() [libsolv/src/repopage.c] - close() - -Namely, when dnf.Base.reset() [dnf/dnf/base.py] is called with (sack=True, -goal=True), the reference counts of the objects pointed to by the "_sack", -"_goal" and "_transaction" fields are supposed to reach zero, and then, as -part of the DnfSack object's finalization, the libsolv file descriptors -are supposed to be closed. - -Now, while this *may* happen immediately in dnf.Base.reset(), it may as -well not. The reason is that there is a multitude of *circular references* -between DnfSack and the packages that it contains. When dnf.Base.reset() -is entered, we have the following picture: - - _sack _goal - | | - v v - +----------------+ +-------------+ - | DnfSack object | <--- | Goal object | - +----------------+ +-------------+ - |^ |^ |^ - || || || - || || || - +--||----||----||---+ - | v| v| v| | <-- _transaction - | Pkg1 Pkg2 PkgN | - | | - | Transaction oject | - +-------------------+ - -That is, the reference count of the DnfSack object is (1 + 1 + N), where N -is the number of packages in the transaction. Details: - -(a) The first reference comes from the "_sack" field, established like - this: - - main [livecd-tools/tools/livecd-creator] - install() [livecd-tools/imgcreate/creator.py] - fill_sack() [dnf/dnf/base.py] - _build_sack() [dnf/dnf/sack.py] - Sack() - sack_init() [libdnf/python/hawkey/sack-py.cpp] - dnf_sack_new() [libdnf/libdnf/dnf-sack.cpp] - -(b) The second reference on the DnfSack object comes from "_goal": - - main [livecd-tools/tools/livecd-creator] - install() [livecd-tools/imgcreate/creator.py] - fill_sack() [dnf/dnf/base.py] - _goal = Goal(_sack) - goal_init() [libdnf/python/hawkey/goal-py.cpp] - Py_INCREF(_sack) - -(c) Then there is one reference to "_sack" *per package* in the - transaction: - - main [livecd-tools/tools/livecd-creator] - install() [livecd-tools/imgcreate/creator.py] - runInstall() [livecd-tools/imgcreate/dnfinst.py] - resolve() [dnf/dnf/base.py] - _goal2transaction() [dnf/dnf/base.py] - list_installs() [libdnf/python/hawkey/goal-py.cpp] - list_generic() [libdnf/python/hawkey/goal-py.cpp] - packagelist_to_pylist() [libdnf/python/hawkey/iutil-py.cpp] - new_package() [libdnf/python/hawkey/sack-py.cpp] - Py_BuildValue() - ts.add_install() - - list_installs() creates a list of packages that need to be installed - by DNF. Inside the loop in packagelist_to_pylist(), which constructs - the elements of that list, Py_BuildValue() is called with the "O" - format specifier, and that increases the reference count on "_sack". - - Subsequently, in the _goal2transaction() method, we iterate over the - package list created by list_installs(), and add each package to the - transaction (ts.add_install()). After _goal2transaction() returns, - this transaction is assigned to "self._transaction" in resolve(). This - is where the last N (back-)references on the DnfSack object come from. - -(d) Now, to quote the defintion of the DnfSack object - ("libdnf/docs/hawkey/tutorial-py.rst"): - -> *Sack* is an abstraction for a collection of packages. - - That's why the DnfSack object references all the Pkg1 through PkgN - packages. - -So, when the dnf.Base.reset() method completes, the picture changes like -this: - - _sack _goal - | | - -- [CUT] -- -- [CUT] -- - | | - v | v - +----------------+ [C] +-------------+ - | DnfSack object | <-[U]- | Goal object | - +----------------+ [T] +-------------+ - |^ |^ |^ | - || || || - || || || | - +--||----||----||---+ [C] - | v| v| v| | <--[U]-- _transaction - | Pkg1 Pkg2 PkgN | [T] - | | | - | Transaction oject | - +-------------------+ - -and we are left with N reference cycles (one between each package and the -same DnfSack object). - -This set of cycles can only be cleaned up by Python's generational garbage -collector . The GC will -collect the DnfSack object, and consequently close the libsolv page file -descriptors via dnf_sack_finalize() -- but garbage collection will happen -*only eventually*, unpredictably. - -This means that the dnf.Base.reset() method breaks its interface contract: - -> Make the Base object forget about various things. - -because the libsolv file descriptors can (and frequently do, in practice) -survive dnf.Base.reset(). - -In general, as long as the garbage collector only tracks process-private -memory blocks, there's nothing wrong; however, file descriptors are -visible to the kernel. When dnf.Base.reset() *temporarily* leaks file -descriptors as explained above, then immediately subsequent operations -that depend on those file descriptors having been closed, can fail. - -An example is livecd-creator's unmounting of: - - /var/tmp/imgcreate-mytcghah/install_root/var/cache/dnf - -which the kernel refuses, due to libsolv's still open file descriptors -pointing into that filesystem: - -> umount: /var/tmp/imgcreate-mytcghah/install_root/var/cache/dnf: target -> is busy. -> Unable to unmount /var/tmp/imgcreate-mytcghah/install_root/var/cache/dnf -> normally, using lazy unmount - -(Unfortunately, the whole lazy umount idea is misguided in livecd-tools; -it's a misfeature that should be removed, as it permits the corruption of -the loop-backed filesystem. Now that the real bug is being fixed in DNF, -lazy umount is not needed as a (broken) workaround in livecd-tools. But -that's a separate patch for livecd-tools: -.) - -Plug the fd leak by forcing a garbage collection in dnf.Base.reset() -whenever we cut the "_sack", "_goal" and "_transaction" links -- that is, -when the "sack" and "goal" parameters are True. - -Note that precisely due to the unpredictable behavior of the garbage -collector, reproducing the bug may prove elusive. In order to reproduce it -deterministically, through usage with livecd-creator, disabling automatic -garbage collection with the following patch (for livecd-tools) is -sufficient: - -> diff --git a/tools/livecd-creator b/tools/livecd-creator -> index 291de10cbbf9..8d2c740c238b 100755 -> --- a/tools/livecd-creator -> +++ b/tools/livecd-creator -> @@ -31,6 +31,8 @@ from dnf.exceptions import Error as DnfBaseError -> import imgcreate -> from imgcreate.errors import KickstartError -> -> +import gc -> + -> class Usage(Exception): -> def __init__(self, msg = None, no_error = False): -> Exception.__init__(self, msg, no_error) -> @@ -261,5 +263,6 @@ def do_nss_libs_hack(): -> return hack -> -> if __name__ == "__main__": -> + gc.disable() -> hack = do_nss_libs_hack() -> sys.exit(main()) - -Also note that you need to use livecd-tools at git commit 4afde9352e82 or -later, for this fix to make any difference: said commit fixes a different -(independent) bug in livecd-tools that produces identical symptoms, but -from a different origin. In other words, if you don't have commit -4afde9352e82 in your livecd-tools install, then said bug in livecd-tools -will mask this DNF fix. - -Signed-off-by: Laszlo Ersek ---- - dnf/base.py | 41 +++++++++++++++++++++++++++++++++++++++++ - 1 file changed, 41 insertions(+) - -diff --git a/dnf/base.py b/dnf/base.py -index caace028..520574b4 100644 ---- a/dnf/base.py -+++ b/dnf/base.py -@@ -72,6 +72,7 @@ import dnf.transaction - import dnf.util - import dnf.yum.rpmtrans - import functools -+import gc - import hawkey - import itertools - import logging -@@ -569,6 +570,46 @@ class Base(object): - self._comps_trans = dnf.comps.TransactionBunch() - self._transaction = None - self._update_security_filters = [] -+ if sack and goal: -+ # We've just done this, above: -+ # -+ # _sack _goal -+ # | | -+ # -- [CUT] -- -- [CUT] -- -+ # | | -+ # v | v -+ # +----------------+ [C] +-------------+ -+ # | DnfSack object | <-[U]- | Goal object | -+ # +----------------+ [T] +-------------+ -+ # |^ |^ |^ | -+ # || || || -+ # || || || | -+ # +--||----||----||---+ [C] -+ # | v| v| v| | <--[U]-- _transaction -+ # | Pkg1 Pkg2 PkgN | [T] -+ # | | | -+ # | Transaction oject | -+ # +-------------------+ -+ # -+ # At this point, the DnfSack object would be released only -+ # eventually, by Python's generational garbage collector, due to the -+ # cyclic references DnfSack<->Pkg1 ... DnfSack<->PkgN. -+ # -+ # The delayed release is a problem: the DnfSack object may -+ # (indirectly) own "page file" file descriptors in libsolv, via -+ # libdnf. For example, -+ # -+ # sack->priv->pool->repos[1]->repodata[1]->store.pagefd = 7 -+ # sack->priv->pool->repos[1]->repodata[2]->store.pagefd = 8 -+ # -+ # These file descriptors are closed when the DnfSack object is -+ # eventually released, that is, when dnf_sack_finalize() (in libdnf) -+ # calls pool_free() (in libsolv). -+ # -+ # We need that to happen right now, as callers may want to unmount -+ # the filesystems which those file descriptors refer to immediately -+ # after reset() returns. Therefore, force a garbage collection here. -+ gc.collect() - - def _closeRpmDB(self): - """Closes down the instances of rpmdb that could be open.""" --- -2.35.1 - diff --git a/0002-Don-t-use-undocumented-re.template.patch b/0002-Don-t-use-undocumented-re.template.patch deleted file mode 100644 index 4598c35..0000000 --- a/0002-Don-t-use-undocumented-re.template.patch +++ /dev/null @@ -1,36 +0,0 @@ -From 8f304e6baa3ea54bb72d8e7e0f5e4143d16678df Mon Sep 17 00:00:00 2001 -From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= -Date: Thu, 12 May 2022 16:19:00 +0200 -Subject: [PATCH] Don't use undocumented re.template() - -Python 3.11.0b1 removed it: https://github.com/python/cpython/commit/b09184bf05 - -It might be resurrected for a proper deprecation period, but it is going away. - -See https://github.com/python/cpython/issues/92728 - -I've looked at the original commit that introduced this code: 6707f479bb -There is no clear indication that would suggest why re.template was used. ---- - dnf/cli/term.py | 5 ++--- - 1 file changed, 2 insertions(+), 3 deletions(-) - -diff --git a/dnf/cli/term.py b/dnf/cli/term.py -index aa075cfe..7361567a 100644 ---- a/dnf/cli/term.py -+++ b/dnf/cli/term.py -@@ -287,9 +287,8 @@ class Term(object): - render = lambda match: beg + match.group() + end - for needle in needles: - pat = escape(needle) -- if ignore_case: -- pat = re.template(pat, re.I) -- haystack = re.sub(pat, render, haystack) -+ flags = re.I if ignore_case else 0 -+ haystack = re.sub(pat, render, haystack, flags=flags) - return haystack - def sub_norm(self, haystack, beg, needles, **kwds): - """Search the string *haystack* for all occurrences of any --- -2.35.3 - diff --git a/dnf.spec b/dnf.spec index 188babd..b5905bc 100644 --- a/dnf.spec +++ b/dnf.spec @@ -65,17 +65,13 @@ It supports RPMs, modules and comps groups & environments. Name: dnf -Version: 4.12.0 -Release: 2%{?dist} +Version: 4.13.0 +Release: 1%{?dist} Summary: %{pkg_summary} # For a breakdown of the licensing, see PACKAGE-LICENSING License: GPLv2+ URL: https://github.com/rpm-software-management/dnf Source0: %{url}/archive/%{version}/%{name}-%{version}.tar.gz -# Upstream commit which fixes leak of libsolv's page file descriptors. -# https://github.com/rpm-software-management/dnf/commit/5ce5ed1ea08ad6e198c1c1642c4d9ea2db6eab86 -Patch0001: 0001-Base.reset-plug-temporary-leak-of-libsolv-s-page-fil.patch -Patch0002: 0002-Don-t-use-undocumented-re.template.patch BuildArch: noarch BuildRequires: cmake BuildRequires: gettext @@ -363,6 +359,12 @@ popd %{python3_sitelib}/%{name}/automatic/ %changelog +* Mon May 30 2022 Jaroslav Rohel - 4.13.0-1 +- Update to 4.13.0 +- Base.reset: plug (temporary) leak of libsolv's page file descriptors +- Small change to better present the option +- Use sqlite cache to make bash completion snappier (RhBug:1815895) + * Fri May 13 2022 Marek Blaha - 4.12.0-2 - Backport patch to not use re.template() deprecated in Python 3.11 diff --git a/sources b/sources index 0912df2..da12603 100644 --- a/sources +++ b/sources @@ -1 +1 @@ -SHA512 (dnf-4.12.0.tar.gz) = 27913bce6a5251d2f7aee0122e4fb238fa6ca8906027f15108a84ceab1cfc40eb5975ac3b365506580f2ff4daedbbfa2738bde2b6107bd2fa3efc2de5dd7a129 +SHA512 (dnf-4.13.0.tar.gz) = 58ab1c4d71ec1d29ca2f2243a3a2caa388415c803382f6c4c2424887a48f048120e269744fd26eb4e7146153415683b40022dadd4ad6ae433bf657b4d9eaefe3