store-api.hh

Go to the documentation of this file.

#pragma once
 
#include "lix/libutil/archive.hh"
#include "lix/libutil/async-io.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/json-fwd.hh"
#include "lix/libutil/logging.hh"
#include "lix/libstore/nar-info.hh"
#include "lix/libstore/realisation.hh"
#include "lix/libstore/path.hh"
#include "lix/libstore/derived-path.hh"
#include "lix/libutil/hash.hh"
#include "lix/libstore/content-address.hh"
#include "lix/libutil/result.hh"
#include "lix/libutil/serialise.hh"
#include "lix/libutil/lru-cache.hh"
#include "lix/libutil/sync.hh"
#include "lix/libstore/globals.hh"
#include "lix/libutil/config.hh"
#include "lix/libutil/json-fwd.hh"
#include "lix/libstore/path-info.hh"
#include "lix/libutil/repair-flag.hh"
#include "lix/libutil/source-path.hh"
 
#include <kj/async.h>
#include <atomic>
#include <limits>
#include <map>
#include <type_traits>
#include <unordered_map>
#include <unordered_set>
#include <memory>
#include <string>
#include <chrono>
#include <variant>
 
 
namespace nix {

 
MakeError(SubstError, Error);
MakeError(BuildError, Error);
MakeError(InvalidPath, Error);
MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
MakeError(BadStorePath, Error);
 
MakeError(InvalidStoreURI, Error);
 
struct BasicDerivation;
struct Derivation;
class FSAccessor;
class NarInfoDiskCache;
class Store;
 
 
typedef std::map<std::string, StorePath> OutputPathMap;
 
 
enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true };
enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true };
enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true };

const uint32_t exportMagic = 0x4558494e;
 
 
enum BuildMode { bmNormal, bmRepair, bmCheck };
BuildMode buildModeFromInteger(int);
 
enum TrustedFlag : bool { NotTrusted = false, Trusted = true };
 
template<>
struct json::is_integral_enum<TrustedFlag> : std::true_type {};
 
struct BuildResult;
struct KeyedBuildResult;
 
 
typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
 
struct StoreConfig : public Config
{
    typedef std::map<std::string, std::string> Params;
 
    using Config::Config;
 
    StoreConfig() = delete;
 
    static StringSet getDefaultSystemFeatures();
 
    virtual ~StoreConfig() noexcept(false) { }

    virtual const std::string name() = 0;

    virtual std::string doc()
    {
        return "";
    }

    virtual std::optional<ExperimentalFeature> experimentalFeature() const
    {
        return std::nullopt;
    }
 
    const PathsSetting<Path> storeDir_{this, settings.nixStore,
        "store",
        R"(
          Logical location of the Nix store, usually
          `/nix/store`. Note that you can only copy store paths
          between stores if they have the same `store` setting.
        )"};
    const Path storeDir = storeDir_;
 
    const Setting<int> pathInfoCacheSize{this, 65536, "path-info-cache-size",
        "Size of the in-memory store path metadata cache."};
 
    const Setting<bool> isTrusted{this, false, "trusted",
        R"(
          Whether paths from this store can be used as substitutes
          even if they are not signed by a key listed in the
          [`trusted-public-keys`](@docroot@/command-ref/conf-file.md#conf-trusted-public-keys)
          setting.
        )"};
 
    Setting<int> priority{this, 0, "priority",
        R"(
          Priority of this store when used as a substituter. A lower value means a higher priority.
        )"};
 
    Setting<bool> wantMassQuery{this, false, "want-mass-query",
        R"(
          Whether this store (when used as a substituter) can be
          queried efficiently for path validity.
        )"};
 
    Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(),
        "system-features",
        "Optional features that the system this store builds on implements (like \"kvm\").",
        {},
        // Don't document the machine-specific default value
        false};
};
 
class Store : public std::enable_shared_from_this<Store>
{
protected:
 
    struct PathInfoCacheValue {

        std::chrono::time_point<std::chrono::steady_clock> time_point = std::chrono::steady_clock::now();

        std::shared_ptr<const ValidPathInfo> value;

        bool isKnownNow();

        inline bool didExist() {
          return value != nullptr;
        }
    };
 
    struct State
    {
        LRUCache<std::string, PathInfoCacheValue> pathInfoCache;
    };
 
    Sync<State, AsyncMutex> state;
 
    std::shared_ptr<NarInfoDiskCache> diskCache;
 
    Store(const StoreConfig & config);
 
public:
    struct AssociatedCredentials
    {
        uid_t user;
        gid_t group;
    };

    virtual std::optional<AssociatedCredentials> associatedCredentials() const
    {
        return {};
    }

    virtual kj::Promise<Result<void>> init()
    {
        return {result::success()};
    }
 
    virtual StoreConfig & config() = 0;
    virtual const StoreConfig & config() const = 0;
 
    virtual ~Store() noexcept(false) { }
 
    virtual std::string getUri() = 0;
 
    StorePath parseStorePath(std::string_view path) const;
 
    std::optional<StorePath> maybeParseStorePath(std::string_view path) const;
 
    std::string printStorePath(const StorePath & path) const;

    StorePathSet parseStorePathSet(const PathSet & paths) const;
 
    PathSet printStorePathSet(const StorePathSet & path) const;

    std::string showPaths(const StorePathSet & paths);

    bool isInStore(PathView path) const;

    bool isStorePath(std::string_view path) const;

    std::pair<StorePath, Path> toStorePath(PathView path) const;

    Path followLinksToStore(std::string_view path) const;

    StorePath followLinksToStorePath(std::string_view path) const;

    StorePath makeStorePath(std::string_view type,
        std::string_view hash, std::string_view name) const;
    StorePath makeStorePath(std::string_view type,
        const Hash & hash, std::string_view name) const;
 
    StorePath makeOutputPath(std::string_view id,
        const Hash & hash, std::string_view name) const;
 
    StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const;
 
    StorePath makeTextPath(std::string_view name, const TextInfo & info) const;
 
    StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const;

    StorePath
    computeStorePathForPathRecursive(std::string_view name, const PreparedDump & source) const;
    StorePath computeStorePathForPathFlat(std::string_view name, const Path & srcPath) const;

    StorePath computeStorePathForText(
        std::string_view name,
        std::string_view s,
        const StorePathSet & references) const;

    kj::Promise<Result<bool>> isValidPath(const StorePath & path);
 
protected:
 
    virtual kj::Promise<Result<bool>> isValidPathUncached(const StorePath & path);
 
public:

    kj::Promise<Result<void>> substitutePaths(const StorePathSet & paths);

    virtual kj::Promise<Result<StorePathSet>> queryValidPaths(const StorePathSet & paths,
        SubstituteFlag maybeSubstitute = NoSubstitute);

    virtual kj::Promise<Result<StorePathSet>> queryAllValidPaths()
    try { unsupported("queryAllValidPaths"); } catch (...) { return {result::current_exception()}; }
 
    constexpr static const char * MissingName = "x";

    kj::Promise<Result<ref<const ValidPathInfo>>> queryPathInfo(const StorePath & path);

    kj::Promise<Result<std::shared_ptr<const Realisation>>> queryRealisation(const DrvOutput &);
 

    virtual bool pathInfoIsUntrusted(const ValidPathInfo &)
    {
        return true;
    }
 
    virtual bool realisationIsUntrusted(const Realisation & )
    {
        return true;
    }
 
protected:

    virtual kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
    queryPathInfoUncached(const StorePath & path) = 0;
    virtual kj::Promise<Result<std::shared_ptr<const Realisation>>>
    queryRealisationUncached(const DrvOutput &) = 0;
 
public:

    virtual kj::Promise<Result<void>>
    queryReferrers(const StorePath & path, StorePathSet & referrers)
    try { unsupported("queryReferrers"); } catch (...) { return {result::current_exception()}; }

    virtual kj::Promise<Result<StorePathSet>> queryValidDerivers(const StorePath & path)
    {
        return {StorePathSet{}};
    }

    virtual kj::Promise<Result<StorePathSet>> queryDerivationOutputs(const StorePath & path);

    virtual kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
    queryPartialDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr);

    virtual kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
    queryStaticPartialDerivationOutputMap(const StorePath & path);

    kj::Promise<Result<OutputPathMap>>
    queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr);

    virtual kj::Promise<Result<std::optional<StorePath>>>
    queryPathFromHashPart(const std::string & hashPart) = 0;

    virtual kj::Promise<Result<StorePathSet>> querySubstitutablePaths(const StorePathSet & paths)
    {
        return {StorePathSet{}};
    }

    virtual kj::Promise<Result<void>> querySubstitutablePathInfos(const StorePathCAMap & paths,
        SubstitutablePathInfos & infos);

    virtual kj::Promise<Result<void>> addToStore(
        const ValidPathInfo & info,
        AsyncInputStream & narSource,
        RepairFlag repair = NoRepair,
        CheckSigsFlag checkSigs = CheckSigs
    ) = 0;

    using PathsSource = std::vector<
        std::pair<ValidPathInfo, std::function<kj::Promise<Result<box_ptr<AsyncInputStream>>>()>>>;

    virtual kj::Promise<Result<void>> addMultipleToStore(
        PathsSource & pathsToCopy,
        Activity & act,
        RepairFlag repair = NoRepair,
        CheckSigsFlag checkSigs = CheckSigs);

    virtual kj::Promise<Result<StorePath>> addToStoreRecursive(
        std::string_view name,
        const PreparedDump & source,
        HashType hashAlgo = HashType::SHA256,
        RepairFlag repair = NoRepair);
    virtual kj::Promise<Result<StorePath>> addToStoreFlat(
        std::string_view name,
        const Path & srcPath,
        HashType hashAlgo = HashType::SHA256,
        RepairFlag repair = NoRepair);

    kj::Promise<Result<ValidPathInfo>> addToStoreSlow(std::string_view name, const Path & srcPath,
        FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = HashType::SHA256,
        std::optional<Hash> expectedCAHash = {});

    virtual kj::Promise<Result<StorePath>> addToStoreFromDump(
        AsyncInputStream & dump,
        std::string_view name,
        FileIngestionMethod method = FileIngestionMethod::Recursive,
        HashType hashAlgo = HashType::SHA256,
        RepairFlag repair = NoRepair,
        const StorePathSet & references = StorePathSet()
    )
    try { unsupported("addToStoreFromDump"); } catch (...) { return {result::current_exception()}; }

    virtual kj::Promise<Result<StorePath>> addTextToStore(
        std::string_view name,
        std::string_view s,
        const StorePathSet & references,
        RepairFlag repair = NoRepair) = 0;

    virtual kj::Promise<Result<void>> registerDrvOutput(const Realisation & output)
    try { unsupported("registerDrvOutput"); } catch (...) { return {result::current_exception()}; }
    virtual kj::Promise<Result<void>>
    registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs)
    {
        return registerDrvOutput(output);
    }

    virtual kj::Promise<Result<box_ptr<Source>>> narFromPath(const StorePath & path) = 0;

    virtual kj::Promise<Result<void>> buildPaths(
        const std::vector<DerivedPath> & paths,
        BuildMode buildMode = bmNormal,
        std::shared_ptr<Store> evalStore = nullptr);

    virtual kj::Promise<Result<std::vector<KeyedBuildResult>>> buildPathsWithResults(
        const std::vector<DerivedPath> & paths,
        BuildMode buildMode = bmNormal,
        std::shared_ptr<Store> evalStore = nullptr);

    virtual kj::Promise<Result<BuildResult>> buildDerivation(
        const StorePath & drvPath,
        const BasicDerivation & drv,
        BuildMode buildMode = bmNormal
    );

    virtual kj::Promise<Result<void>> ensurePath(const StorePath & path);

    virtual kj::Promise<Result<void>> addTempRoot(const StorePath & path)
    try {
        debug("not creating temporary root, store doesn't support GC");
        return {result::success()};
    } catch (...) {
        return {result::current_exception()};
    }

    kj::Promise<Result<std::string>> makeValidityRegistration(const StorePathSet & paths,
        bool showDerivers, bool showHash);

    kj::Promise<Result<JSON>> pathInfoToJSON(const StorePathSet & storePaths,
        bool includeImpureInfo, bool showClosureSize,
        Base hashBase = Base::Base32,
        AllowInvalidFlag allowInvalid = DisallowInvalid);

    kj::Promise<Result<std::pair<uint64_t, uint64_t>>> getClosureSize(const StorePath & storePath);

    virtual kj::Promise<Result<void>> optimiseStore() { return {result::success()}; }

    virtual kj::Promise<Result<bool>> verifyStore(bool checkContents, RepairFlag repair = NoRepair)
    {
        return {false};
    }

    virtual ref<FSAccessor> getFSAccessor() = 0;

    virtual kj::Promise<Result<void>> repairPath(const StorePath & path);

    virtual kj::Promise<Result<void>>
    addSignatures(const StorePath & storePath, const StringSet & sigs)
    try { unsupported("addSignatures"); } catch (...) { return {result::current_exception()}; }
 
    /* Utility functions. */

    kj::Promise<Result<Derivation>> derivationFromPath(const StorePath & drvPath);

    kj::Promise<Result<Derivation>> readDerivation(const StorePath & drvPath);

    kj::Promise<Result<Derivation>> readInvalidDerivation(const StorePath & drvPath);

    virtual kj::Promise<Result<void>> computeFSClosure(const StorePathSet & paths,
        StorePathSet & out, bool flipDirection = false,
        bool includeOutputs = false, bool includeDerivers = false);
 
    kj::Promise<Result<void>> computeFSClosure(const StorePath & path,
        StorePathSet & out, bool flipDirection = false,
        bool includeOutputs = false, bool includeDerivers = false);

    virtual kj::Promise<Result<void>> queryMissing(const std::vector<DerivedPath> & targets,
        StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
        uint64_t & downloadSize, uint64_t & narSize);

    kj::Promise<Result<StorePaths>> topoSortPaths(const StorePathSet & paths);

    kj::Promise<Result<void>> exportPaths(const StorePathSet & paths, Sink & sink);
 
    kj::Promise<Result<void>> exportPath(const StorePath & path, Sink & sink);

    kj::Promise<Result<StorePaths>>
    importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs);
 
    template<template<typename> typename Wrapper = std::type_identity_t>
    struct Stats
    {
        Wrapper<uint64_t> narInfoRead{0};
        Wrapper<uint64_t> narInfoReadAverted{0};
        Wrapper<uint64_t> narInfoMissing{0};
        Wrapper<uint64_t> narInfoWrite{0};
        Wrapper<uint64_t> pathInfoCacheSize{0};
        Wrapper<uint64_t> narRead{0};
        Wrapper<uint64_t> narReadBytes{0};
        Wrapper<uint64_t> narReadCompressedBytes{0};
        Wrapper<uint64_t> narWrite{0};
        Wrapper<uint64_t> narWriteAverted{0};
        Wrapper<uint64_t> narWriteBytes{0};
        Wrapper<uint64_t> narWriteCompressedBytes{0};
        Wrapper<uint64_t> narWriteCompressionTimeMs{0};
    };
 
    kj::Promise<Result<Stats<>>> getStats();

    kj::Promise<Result<StorePathSet>>
    exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths);

    kj::Promise<Result<std::optional<StorePath>>> getBuildDerivationPath(const StorePath &);

    kj::Promise<void> clearPathInfoCache()
    {
        (co_await state.lock())->pathInfoCache.clear();
    }

    virtual kj::Promise<Result<void>> connect() { return {result::success()}; }

    virtual kj::Promise<Result<unsigned int>> getProtocol()
    {
        return {result::success(0)};
    };

    virtual kj::Promise<Result<std::optional<TrustedFlag>>> isTrustedClient() = 0;
 
 
    virtual Path toRealPath(const Path & storePath)
    {
        return storePath;
    }
 
    Path toRealPath(const StorePath & storePath)
    {
        return toRealPath(printStorePath(storePath));
    }

    virtual kj::Promise<Result<void>> setOptions() { return {result::success()}; }
 
    virtual kj::Promise<Result<std::optional<std::string>>> getVersion()
    {
        return {result::success(std::nullopt)};
    }
 
protected:
 
    Stats<std::atomic> stats;

    [[noreturn]] void unsupported(const std::string & op)
    {
        throw Unsupported("operation '%s' is not supported by store '%s'", op, getUri());
    }
 
};
 

kj::Promise<Result<void>> copyStorePath(
    Store & srcStore,
    Store & dstStore,
    const StorePath & storePath,
    RepairFlag repair = NoRepair,
    CheckSigsFlag checkSigs = CheckSigs);
 

kj::Promise<Result<std::map<StorePath, StorePath>>> copyPaths(
    Store & srcStore, Store & dstStore,
    const RealisedPath::Set &,
    RepairFlag repair = NoRepair,
    CheckSigsFlag checkSigs = CheckSigs,
    SubstituteFlag substitute = NoSubstitute);
 
kj::Promise<Result<std::map<StorePath, StorePath>>> copyPaths(
    Store & srcStore, Store & dstStore,
    const StorePathSet & paths,
    RepairFlag repair = NoRepair,
    CheckSigsFlag checkSigs = CheckSigs,
    SubstituteFlag substitute = NoSubstitute);

kj::Promise<Result<void>> copyClosure(
    Store & srcStore, Store & dstStore,
    const RealisedPath::Set & paths,
    RepairFlag repair = NoRepair,
    CheckSigsFlag checkSigs = CheckSigs,
    SubstituteFlag substitute = NoSubstitute);
 
kj::Promise<Result<void>> copyClosure(
    Store & srcStore, Store & dstStore,
    const StorePathSet & paths,
    RepairFlag repair = NoRepair,
    CheckSigsFlag checkSigs = CheckSigs,
    SubstituteFlag substitute = NoSubstitute);

void removeTempRoots();
 

kj::Promise<Result<StorePath>>
resolveDerivedPath(Store &, const SingleDerivedPath &, Store * evalStore = nullptr);
kj::Promise<Result<OutputPathMap>>
resolveDerivedPath(Store &, const DerivedPath::Built &, Store * evalStore = nullptr);

kj::Promise<Result<ref<Store>>> openStore(const std::string & uri = settings.storeUri.get(),
    const StoreConfig::Params & extraParams = {});
 

kj::Promise<Result<std::list<ref<Store>>>> getDefaultSubstituters();
 
struct StoreFactory
{
    std::set<std::string> uriSchemes;
    std::function<
        std::optional<ref<Store>> (
            const std::string & scheme,
            const std::string & uri,
            const StoreConfig::Params & params
        )
    > create;
    std::function<std::shared_ptr<StoreConfig> ()> getConfig;
};
 
struct StoreImplementations
{
    static std::vector<StoreFactory> * registered;
 
    template<typename T, typename TConfig>
    static void add()
    {
        if (!registered) registered = new std::vector<StoreFactory>();
        StoreFactory factory{
            .uriSchemes = T::uriSchemes(),
            .create =
                ([](const std::string & scheme, const std::string & uri, const StoreConfig::Params & params)
                 -> std::optional<ref<Store>>
                 { return make_ref<T>(scheme, uri, params); }),
            .getConfig =
                ([]()
                 -> std::shared_ptr<StoreConfig>
                 { return std::make_shared<TConfig>(StringMap({})); })
        };
        registered->push_back(factory);
    }
};
 

std::string showPaths(const PathSet & paths);
 
 
std::optional<ValidPathInfo> decodeValidPathInfo(
    const Store & store,
    std::istream & str,
    std::optional<HashResult> hashGiven = std::nullopt);

std::pair<std::string, StoreConfig::Params> splitUriAndParams(const std::string & uri);
 
const ContentAddress * getDerivationCA(const BasicDerivation & drv);
 
kj::Promise<Result<std::map<DrvOutput, StorePath>>> drvOutputReferences(
    Store & store,
    const Derivation & drv,
    const StorePath & outputPath,
    Store * evalStore = nullptr);
 
}