Input::getAccessorUnchecked(): Wrap fetches in a path lock

edolstra · edolstra · commit 57ea531f74c4 · 2026-04-01T15:47:56.000+02:00
This prevents multiple processes (like nix-eval-jobs instances) from
fetching the same input at the same time. That doesn't matter for
correctness, but it can cause a lot of redundant downloads.
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
@@ -8,6 +8,9 @@
 #include "nix/util/url.hh"
 #include "nix/util/forwarding-source-accessor.hh"
 #include "nix/util/archive.hh"
+#include "nix/util/users.hh"
+#include "nix/store/pathlocks.hh"
+#include "nix/util/environment-variables.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -367,6 +370,18 @@ std::pair<ref<SourceAccessor>, Input> Input::getAccessorUnchecked(const Settings
         return {accessor, result};
     };
 
+    /* Acquire a path lock on this input. Note that fetching the same input in parallel is supposed to be safe (it's up
+     * to the fetchers to guarantee this), so this is merely intended to avoid work duplication. */
+    auto lockFilePath =
+        getCacheDir() / "fetcher-locks"
+        / hashString(HashAlgorithm::SHA256, attrsToJSON(toAttrs()).dump()).to_string(HashFormat::Base16, false);
+    std::filesystem::create_directories(lockFilePath.parent_path());
+    PathLocks lock(
+        {lockFilePath.string()}, fmt("waiting for another Nix process to finish fetching input '%s'...", to_string()));
+
+    if (getEnv("_NIX_TEST_CONCURRENT_FETCHES"))
+        std::this_thread::sleep_for(std::chrono::seconds(1));
+
     /* See if the input is in the cache of the fetcher. */
     try {
         if (auto res = scheme->getAccessor(settings, store, *this, true))
diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh
@@ -115,3 +115,17 @@ path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar"
 [[ $(cat "$path/a/b/xyzzy") = xyzzy ]]
 [[ $(cat "$path/a/b/foo") = foo ]]
 [[ $(cat "$path/bla") = abc ]]
+
+# Test that concurrent invocations of Nix will fetch the tarball only once.
+clearStore
+rm -rf "$TEST_HOME/.cache"
+nix-store --init # needed because concurrent creation of the store can give SQLite errors
+_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log1" &
+pid1="$!"
+_NIX_TEST_CONCURRENT_FETCHES=1 _NIX_FORCE_HTTP=1 nix flake prefetch -v "tarball+file://$TEST_ROOT/tar.tar" 2> "$TEST_ROOT/log2" &
+pid2="$!"
+wait "$pid1"
+wait "$pid2"
+[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "Download.*to") -eq 2 ]]
+[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "downloading.*tar.tar") -eq 1 ]]
+[[ $(cat "$TEST_ROOT/log1" "$TEST_ROOT/log2" | grep -c "waiting for another Nix process to finish fetching input") -eq 1 ]]
