local-store.hh

Go to the documentation of this file.

#pragma once
 
#include "lix/libstore/sqlite.hh"
 
#include "lix/libstore/store-api.hh"
#include "lix/libstore/indirect-root-store.hh"
#include "lix/libutil/async-io.hh"
#include "lix/libutil/sync.hh"
#include "lix/libutil/types.hh"
 
#include <chrono>
#include <future>
#include <kj/async.h>
#include <string>
#include <mutex>
#include <memory>
#include <unordered_set>
 
 
namespace nix {
 

const int nixSchemaVersion = 10;
 
 
struct OptimiseStats
{
    unsigned long filesLinked = 0;
    uint64_t bytesFreed = 0;
    uint64_t blocksFreed = 0;
};
 
struct LocalStoreConfig final : LocalFSStoreConfig
{
    using LocalFSStoreConfig::LocalFSStoreConfig;
 
    Setting<bool> requireSigs{this,
        settings.requireSigs,
        "require-sigs",
        "Whether store paths copied into this store should have a trusted signature."};
 
    Setting<bool> readOnly{this,
        false,
        "read-only",
        R"(
          Allow this store to be opened when its [database](@docroot@/glossary.md#gloss-nix-database) is on a read-only filesystem.
 
          Normally Lix will attempt to open the store database in read-write mode, even for querying (when write access is not needed), causing it to fail if the database is on a read-only filesystem.
 
          Enable read-only mode to disable locking and open the SQLite database with the [`immutable` parameter](https://www.sqlite.org/c3ref/open.html) set.
 
          > **Warning**
          > Do not use this unless the filesystem is read-only.
          >
          > Using it when the filesystem is writable can cause incorrect query results or corruption errors if the database is changed by another process.
          > While the filesystem the database resides on might appear to be read-only, consider whether another user or system might have write access to it.
        )"};
 
    const std::string name() override { return "Local Store"; }
 
    std::string doc() override;
};
 
class LocalStore : public virtual IndirectRootStore
    , public virtual GcStore
{
private:
 
    LocalStoreConfig config_;

    AutoCloseFD globalLock;

    std::unique_ptr<const PublicKeys> publicKeys = nullptr;
    std::once_flag publicKeysFlag;
 
    struct DBState
    {
        SQLite db;
 
        struct Stmts;
        std::unique_ptr<Stmts> stmts;
    };
 
    Sync<DBState, AsyncMutex> _dbState;
 
    struct GCState
    {
        std::chrono::time_point<std::chrono::steady_clock> lastGCCheck;

        bool gcRunning = false;
        std::future<void> gcFuture;
        std::list<kj::Own<kj::CrossThreadPromiseFulfiller<void>>> gcWaiters;

        uint64_t availAfterGC = std::numeric_limits<uint64_t>::max();
    };
 
    Sync<GCState> _gcState;
 
    std::optional<AssociatedCredentials> association;
 
public:
 
    const Path dbDir;
    const Path linksDir;
    const Path reservedSpacePath;
    const Path schemaPath;
    const Path tempRootsDir;
    const Path fnTempRoots;
 
    LocalStoreConfig & config() override { return config_; }
    const LocalStoreConfig & config() const override { return config_; }
 
    std::optional<AssociatedCredentials> associatedCredentials() const override
    {
        return association;
    }
 
    void associateWithCredentials(uid_t user, gid_t group)
    {
        association = {user, group};
    }
 
private:
 
    const PublicKeys & getPublicKeys();
 
protected:

    LocalStore(LocalStoreConfig config);
    LocalStore(std::string scheme, std::string path, LocalStoreConfig config);
 
public:

    PathSet locksHeld;
 
    virtual ~LocalStore();
 
    static std::set<std::string> uriSchemes()
    { return {}; }

    static ref<LocalStore> makeLocalStore(const StoreConfig::Params & params);

 
    std::string getUri() override;
 
    kj::Promise<Result<bool>> isValidPathUncached(const StorePath & path) override;
 
    kj::Promise<Result<StorePathSet>> queryValidPaths(const StorePathSet & paths,
        SubstituteFlag maybeSubstitute = NoSubstitute) override;
 
    kj::Promise<Result<StorePathSet>> queryAllValidPaths() override;
 
    kj::Promise<Result<std::shared_ptr<const ValidPathInfo>>>
    queryPathInfoUncached(const StorePath & path) override;
 
    kj::Promise<Result<void>>
    queryReferrers(const StorePath & path, StorePathSet & referrers) override;
 
    kj::Promise<Result<StorePathSet>> queryValidDerivers(const StorePath & path) override;
 
    kj::Promise<Result<std::map<std::string, std::optional<StorePath>>>>
    queryStaticPartialDerivationOutputMap(const StorePath & path) override;
 
    kj::Promise<Result<std::optional<StorePath>>>
    queryPathFromHashPart(const std::string & hashPart) override;
 
    kj::Promise<Result<StorePathSet>> querySubstitutablePaths(const StorePathSet & paths) override;
 
    bool pathInfoIsUntrusted(const ValidPathInfo &) override;
    bool realisationIsUntrusted(const Realisation & ) override;
 
    kj::Promise<Result<void>> addToStore(const ValidPathInfo & info, AsyncInputStream & source,
        RepairFlag repair, CheckSigsFlag checkSigs) override;
 
    kj::Promise<Result<StorePath>> addToStoreFromDump(
        AsyncInputStream & dump,
        std::string_view name,
        FileIngestionMethod method,
        HashType hashAlgo,
        RepairFlag repair,
        const StorePathSet & references
    ) override;
 
    kj::Promise<Result<StorePath>> addTextToStore(
        std::string_view name,
        std::string_view s,
        const StorePathSet & references,
        RepairFlag repair) override;
 
    kj::Promise<Result<void>> addTempRoot(const StorePath & path) override;
 
private:
 
    void createTempRootsFile();

    Sync<AutoCloseFD> _fdTempRoots;

    Sync<AutoCloseFD> _fdGCLock;

    Sync<AutoCloseFD> _fdRootsSocket;
 
public:

    kj::Promise<Result<void>> addIndirectRoot(const Path & path) override;
 
private:
 
    void findTempRoots(Roots & roots, bool censor);
 
    AutoCloseFD openGCLock();
 
public:
 
    kj::Promise<Result<Roots>> findRoots(bool censor) override;
 
    kj::Promise<Result<void>>
    collectGarbage(const GCOptions & options, GCResults & results) override;

    kj::Promise<Result<void>> optimiseStore(OptimiseStats & stats);
 
    kj::Promise<Result<void>> optimiseStore() override;

    void optimisePath(const Path & path, RepairFlag repair);
 
    kj::Promise<Result<bool>> verifyStore(bool checkContents, RepairFlag repair) override;

    kj::Promise<Result<void>> registerValidPath(const ValidPathInfo & info);
 
    kj::Promise<Result<void>> registerValidPaths(const ValidPathInfos & infos);
 
    kj::Promise<Result<unsigned int>> getProtocol() override;
 
    kj::Promise<Result<std::optional<TrustedFlag>>> isTrustedClient() override;
 
    kj::Promise<Result<void>>
    addSignatures(const StorePath & storePath, const StringSet & sigs) override;

    kj::Promise<Result<void>> autoGC(bool sync = true);

    kj::Promise<Result<void>> registerDrvOutput(const Realisation & info) override;
    kj::Promise<Result<void>>
    registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
    void cacheDrvOutputMapping(
        DBState & state,
        const uint64_t deriver,
        const std::string & outputName,
        const StorePath & output);
 
    std::optional<const Realisation> queryRealisation_(DBState & state, const DrvOutput & id);
    std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(DBState & state, const DrvOutput & id);
    kj::Promise<Result<std::shared_ptr<const Realisation>>>
    queryRealisationUncached(const DrvOutput&) override;
 
    kj::Promise<Result<std::optional<std::string>>> getVersion() override;
 
private:

    int getSchema();
 
    void initDB(DBState & state);
    void openDB(DBState & state, bool create);
    void prepareStatements(DBState & state);
 
    void makeStoreWritable();
 
    uint64_t queryValidPathId(DBState & state, const StorePath & path);
 
    kj::Promise<Result<uint64_t>>
    addValidPath(DBState & state, const ValidPathInfo & info, bool checkOutputs = true);
 
    kj::Promise<Result<void>> invalidatePath(DBState & state, const StorePath & path);

    kj::Promise<Result<void>> invalidatePathChecked(const StorePath & path);
 
    kj::Promise<Result<void>> verifyPath(const StorePath & path, const StorePathSet & store,
        StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
 
    std::shared_ptr<const ValidPathInfo> queryPathInfoInternal(DBState & state, const StorePath & path);
 
    void updatePathInfo(DBState & state, const ValidPathInfo & info);
 
    PathSet queryValidPathsOld();
    ValidPathInfo queryPathInfoOld(const Path & path);
 
    kj::Promise<Result<void>> findRoots(const Path & path, unsigned char type, Roots & roots);
 
    kj::Promise<Result<void>> findRootsNoTemp(Roots & roots, bool censor);

    virtual void findPlatformRoots(UncheckedRoots & unchecked);
 
    kj::Promise<Result<void>> findRuntimeRoots(Roots & roots, bool censor);
 
    std::pair<Path, AutoCloseFD> createTempDirInStore();
 
    typedef std::unordered_set<ino_t> InodeHash;
 
    InodeHash loadInodeHash();
    Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash);
    void optimisePath_(Activity * act, OptimiseStats & stats, const Path & path, InodeHash & inodeHash, RepairFlag repair);
 
    // Internal versions that are not wrapped in retry_sqlite.
    bool isValidPath_(DBState & state, const StorePath & path);
    void queryReferrers(DBState & state, const StorePath & path, StorePathSet & referrers);

    void signPathInfo(ValidPathInfo & info);
    void signRealisation(Realisation &);
 
    // XXX: Make a generic `Store` method
    ContentAddress hashCAPath(
        const ContentAddressMethod & method,
        const HashType & hashType,
        const StorePath & path);
 
    ContentAddress hashCAPath(
        const ContentAddressMethod & method,
        const HashType & hashType,
        const Path & path,
        const std::string_view pathHash
    );
 
    kj::Promise<Result<void>> addBuildLog(const StorePath & drvPath, std::string_view log) override;
 
    friend struct LocalDerivationGoal;
    friend struct PathSubstitutionGoal;
    friend struct SubstitutionGoal;
    friend struct DerivationGoal;
};
 
 
typedef std::pair<dev_t, ino_t> Inode;
typedef std::set<Inode> InodesSeen;
 

void canonicalisePathMetaData(
    const Path & path,
    std::optional<std::pair<uid_t, uid_t>> uidRange,
    InodesSeen & inodesSeen);
void canonicalisePathMetaData(
    const Path & path,
    std::optional<std::pair<uid_t, uid_t>> uidRange);
 
void canonicaliseTimestampAndPermissions(const Path & path);
 
MakeError(PathInUse, Error);
 
// Implemented by the relevant platform/ module being used.
void registerLocalStore();
 
}