eval.hh

Go to the documentation of this file.

#pragma once
 
#include "lix/libexpr/attr-set.hh"
#include "lix/libexpr/eval-error.hh"
#include "lix/libexpr/gc-alloc.hh"
#include "lix/libutil/box_ptr.hh"
#include "lix/libutil/generator.hh"
#include "lix/libutil/async.hh"
#include "lix/libutil/source-path.hh"
#include "lix/libutil/types.hh"
#include "lix/libexpr/value.hh"
#include "lix/libexpr/nixexpr.hh"
#include "lix/libexpr/symbol-table.hh"
#include "lix/libutil/config.hh"
#include "lix/libutil/experimental-features.hh"
#include "lix/libexpr/search-path.hh"
#include "lix/libexpr/repl-exit-status.hh"
#include "lix/libutil/backed-string-view.hh"
 
#include <concepts>
#include <map>
#include <optional>
#include <unordered_map>
#include <functional>
 
namespace nix {
 
class Store;
class EvalState;
class StorePath;
struct SingleDerivedPath;
enum RepairFlag : bool;
struct MemoryInputAccessor;
namespace eval_cache {
    class EvalCache;
}

using PrimOpImpl = void(EvalState & state, Value ** args, Value & v);

struct PrimOp
{
    std::string name;

    std::vector<std::string> args;

    size_t arity = 0;

    const char * doc = nullptr;

    std::function<PrimOpImpl> fun;

    std::optional<ExperimentalFeature> experimentalFeature;
};
 
std::ostream & operator<<(std::ostream & output, PrimOp & primOp);

struct Constant
{
    ValueType type = nThunk;

    const char * doc = nullptr;

    bool impureOnly = false;
};
 
using ValMap = GcMap<std::string, Value *>;
 
struct Env
{
    Env * up;
    Value * values[0];
};
 
void printEnvBindings(const EvalState &es, const Expr & expr, const Env & env);
void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env, int lvl = 0);
 
std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env);
 
void copyContext(const Value & v, NixStringContext & context);
 
 
std::string printValue(EvalState & state, Value & v);
std::ostream & operator << (std::ostream & os, const ValueType t);
 

void initLibExpr();
 
 
struct RegexCache;
 
struct DebugTrace {
    std::shared_ptr<Pos> pos;
    const Expr & expr;
    const Env & env;
    HintFmt hint;
    bool isError;
    std::shared_ptr<const DebugTrace> parent;
};
 
struct DebugState
{
private:
    std::weak_ptr<const DebugTrace> latestTrace;
    const PosTable & positions;
    const SymbolTable & symbols;
 
public:
    std::function<ReplExitStatus(ValMap const & extraEnv, NeverAsync)> errorCallback;
    bool stop = false;
    bool inDebugger = false;
    std::map<const Expr *, const std::shared_ptr<const StaticEnv>> exprEnvs;
    int trylevel = 0;
 
    explicit DebugState(
        const PosTable & positions,
        const SymbolTable & symbols,
        std::function<ReplExitStatus(ValMap const & extraEnv, NeverAsync)> errorCallback
    )
        : positions(positions)
        , symbols(symbols)
        , errorCallback(errorCallback)
    {
        assert(errorCallback);
    }
 
    void onEvalError(const EvalError * error, const Env & env, const Expr & expr, NeverAsync = {});
 
    const std::shared_ptr<const StaticEnv> staticEnvFor(const Expr & expr) const
    {
        if (auto i = exprEnvs.find(&expr); i != exprEnvs.end()) {
            return i->second;
        }
        return nullptr;
    }
 
    class TraceFrame
    {
        friend struct DebugState;
        template<std::derived_from<EvalError> T>
        friend class EvalErrorBuilder;
 
        // holds both the data for this frame *and* a deleter that pulls this frame
        // off the trace stack. EvalErrorBuilder uses this for withFrame fake trace
        // frames, and to avoid needing to see this class definition in its header.
        const std::shared_ptr<const DebugTrace> entry = nullptr;
 
        explicit TraceFrame(std::shared_ptr<const DebugTrace> entry): entry(std::move(entry)) {}
 
    public:
        TraceFrame(std::nullptr_t) {}
    };
 
    TraceFrame addTrace(DebugTrace t);

    Generator<const DebugTrace *> traces()
    {
        for (auto current = latestTrace.lock(); current; current = current->parent) {
            co_yield current.get();
        }
    }
};
 
struct StaticSymbols
{
    const Symbol outPath, drvPath, type, meta, name, value, system, overrides, outputs, outputName,
        ignoreNulls, file, line, column, functor, toString, right, wrong, structuredAttrs,
        allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites, maxSize,
        maxClosureSize, builder, args, contentAddressed, impure, outputHash, outputHashAlgo,
        outputHashMode, recurseForDerivations, description, self, startSet, operator_, key,
        path, prefix, outputSpecified;
 
    const Expr::AstSymbols exprSymbols;
 
    explicit StaticSymbols(SymbolTable & symbols);
};
 
class EvalMemory
{
    std::shared_ptr<void *> valueAllocCache;

    std::shared_ptr<void *> env1AllocCache;
 
public:
    struct Statistics
    {
        unsigned long nrEnvs = 0;
        unsigned long nrValuesInEnvs = 0;
        unsigned long nrValues = 0;
        unsigned long nrAttrsets = 0;
        unsigned long nrAttrsInAttrsets = 0;
        unsigned long nrListElems = 0;
    };
 
    EvalMemory();
 
    EvalMemory(const EvalMemory &) = delete;
    EvalMemory(EvalMemory &&) = delete;
    EvalMemory & operator=(const EvalMemory &) = delete;
    EvalMemory & operator=(EvalMemory &&) = delete;
 
    inline Value * allocValue();
    inline Env & allocEnv(size_t size);
 
    Bindings * allocBindings(size_t capacity);
    Value newList(size_t length);
 
    BindingsBuilder buildBindings(SymbolTable & symbols, size_t capacity)
    {
        return BindingsBuilder(*this, symbols, allocBindings(capacity));
    }
 
    const Statistics getStats() const { return stats; }
 
private:
    Statistics stats;
};
 
class EvalBuiltins
{
    EvalMemory & mem;
    SymbolTable & symbols;
 
public:
    explicit EvalBuiltins(
        EvalMemory & mem,
        SymbolTable & symbols,
        const SearchPath & searchPath,
        const Path & storeDir,
        size_t size = 128
    );

    Env & env;

    std::shared_ptr<StaticEnv> staticEnv; // !!! should be private

    std::vector<std::pair<std::string, Constant>> constantInfos;
 
private:
    unsigned int baseEnvDispl = 0;
 
    void createBaseEnv(const SearchPath & searchPath, const Path & storeDir);
 
    Value * addConstant(const std::string & name, const Value & v, Constant info);
 
    void addConstant(const std::string & name, Value * v, Constant info);
 
    Value * addPrimOp(PrimOp && primOp);
 
    Value prepareNixPath(const SearchPath & searchPath);
 
public:
    Value & get(const std::string & name);
 
    struct Doc
    {
        Pos pos;
        std::optional<std::string> name;
        size_t arity;
        std::vector<std::string> args;
        const char * doc;
    };
 
    std::optional<Doc> getDoc(Value & v);
};
 
struct CachedEvalFile;
 
struct EvalRuntimeCaches
{
    RootValue vCallFlake;
    RootValue vImportedDrvToDerivation;

    std::shared_ptr<RegexCache> regexes;

    std::map<SourcePath, std::shared_ptr<CachedEvalFile>> fileEval;
};
 
struct EvalErrorContext
{
    const PosTable & positions;
    DebugState * debug;
 
    template<std::derived_from<EvalError> T, typename... Args>
    [[gnu::noinline]]
    EvalErrorBuilder<T> make(const Args & ... args) {
        return EvalErrorBuilder<T>(positions, debug, args...);
    }
};
 
class EvalPaths
{
    ref<Store> store;
    SearchPath searchPath_;
    EvalErrorContext & errors;
 
public:
    EvalPaths(
        AsyncIoRoot & aio,
        const ref<Store> & store,
        SearchPath searchPath,
        EvalErrorContext & errors
    );
 
    const SearchPath & searchPath() const { return searchPath_; }
 
private:
    struct AllowedPath
    {
        struct ComponentLess : std::less<>
        {
            // we'll only use this for string-likes, it's fine. trust me sis.
            using is_transparent = void;
        };
 
        std::map<std::string, AllowedPath, ComponentLess> children;
 
        bool allowAllChildren = false;
    };

    std::optional<AllowedPath> allowedPaths;
 
 
    /* Cache for calls to addToStore(); maps source paths to the store
       paths. */
    std::map<SourcePath, StorePath> srcToStore;
 
    std::map<std::string, std::optional<std::string>> searchPathResolved;

    std::unordered_map<Path, CheckedSourcePath> resolvedPaths;
 
public:
    void allowPath(const Path & path);

    void allowPath(const StorePath & storePath);

    void allowAndSetStorePathString(const StorePath & storePath, Value & v);

    CheckedSourcePath checkSourcePath(const SourcePath & path);

    CheckedSourcePath resolveExprPath(SourcePath path);
 
    void checkURI(const std::string & uri);

    Path toRealPath(const Path & path, const NixStringContext & context);

    template<typename T, typename E>
    struct PathResult : private std::variant<T, EvalErrorBuilder<E>>
    {
        PathResult(T p) : std::variant<T, EvalErrorBuilder<E>>(std::move(p)) {}
        PathResult(EvalErrorBuilder<E> e) : std::variant<T, EvalErrorBuilder<E>>(std::move(e)) {}
 
        T unwrap(NeverAsync = {}) &&
        {
            return std::visit(
                overloaded{
                    [](T & p) -> T { return std::move(p); },
                    [](EvalErrorBuilder<E> & e) -> T {
                        std::move(e).debugThrow(always_progresses);
                    }
                },
                static_cast<std::variant<T, EvalErrorBuilder<E>> &>(*this)
            );
        }
    };

    kj::Promise<Result<PathResult<SourcePath, ThrownError>>> findFile(const std::string_view path);
    kj::Promise<Result<PathResult<SourcePath, ThrownError>>>
    findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos);

    kj::Promise<Result<std::optional<std::string>>>
    resolveSearchPathPath(const SearchPath::Path & path);
 
    kj::Promise<Result<PathResult<StorePath, EvalError>>> copyPathToStore(
        NixStringContext & context, const SourcePath & path, RepairFlag repair = NoRepair
    );

    void mkStorePathString(const StorePath & storePath, Value & v);
};
 
struct EvalStatistics
{
    unsigned long nrLookups = 0;
    unsigned long nrAvoided = 0;
    unsigned long nrOpUpdates = 0;
    unsigned long nrOpUpdateValuesCopied = 0;
    unsigned long nrListConcats = 0;
    unsigned long nrPrimOpCalls = 0;
    unsigned long nrFunctionCalls = 0;
    unsigned long nrThunks = 0;
 
    bool countCalls = false;
 
    std::map<std::string, size_t> primOpCalls;
    std::map<ExprLambda *, size_t> functionCalls;
    std::map<PosIdx, size_t> attrSelects;
 
    void addCall(ExprLambda & fun);
};
 
class Evaluator
{
    friend class EvalBuiltins;
    friend class EvalState;
 
    EvalState * activeEval = nullptr;
 
public:
    SymbolTable symbols;
    PosTable positions;
    const StaticSymbols s;
    EvalMemory mem;
    EvalRuntimeCaches caches;
    EvalPaths paths;
    EvalBuiltins builtins;
    EvalStatistics stats;

    RepairFlag repair;

    const ref<Store> store;

    ref<Store> buildStore;
 
    std::unique_ptr<DebugState> debug;
    EvalErrorContext errors;
 
    Evaluator(
        AsyncIoRoot & aio,
        const SearchPath & _searchPath,
        ref<Store> store,
        std::shared_ptr<Store> buildStore = nullptr,
        std::function<ReplExitStatus(EvalState & es, ValMap const & extraEnv)> debugRepl = nullptr
    );
 
    Evaluator(const Evaluator &) = delete;
    Evaluator(Evaluator &&) = delete;
    Evaluator & operator=(const Evaluator &) = delete;
    Evaluator & operator=(Evaluator &&) = delete;

    Expr & parseExprFromFile(const CheckedSourcePath & path);
    Expr & parseExprFromFile(const CheckedSourcePath & path, std::shared_ptr<StaticEnv> & staticEnv);

    Expr & parseExprFromString(
        std::string s,
        const SourcePath & basePath,
        std::shared_ptr<StaticEnv> & staticEnv,
        const FeatureSettings & xpSettings = featureSettings
    );
    Expr & parseExprFromString(
        std::string s,
        const SourcePath & basePath,
        const FeatureSettings & xpSettings = featureSettings
    );
 
    std::variant<std::unique_ptr<Expr>, ExprReplBindings>
    parseReplInput(
        std::string s,
        const SourcePath & basePath,
        std::shared_ptr<StaticEnv> & staticEnv,
        const FeatureSettings & xpSettings = featureSettings
    );
 
    Expr & parseStdin();

    void evalLazily(Expr & e, Value & v);
 
private:
    Expr * parse(
        char * text,
        size_t length,
        Pos::Origin origin,
        const SourcePath & basePath,
        std::shared_ptr<StaticEnv> & staticEnv,
        const FeatureSettings & xpSettings = featureSettings);
 
    std::variant<std::unique_ptr<Expr>, ExprReplBindings>
    parse_repl(
        char * text,
        size_t length,
        Pos::Origin origin,
        const SourcePath & basePath,
        std::shared_ptr<StaticEnv> & staticEnv,
        const FeatureSettings & xpSettings = featureSettings
    );
 
public:
    BindingsBuilder buildBindings(size_t capacity)
    {
        return mem.buildBindings(symbols, capacity);
    }

    void maybePrintStats();

    void printStatistics();

    bool fullGC();

    box_ptr<EvalState> begin(AsyncIoRoot & aio);
};
 
 
class EvalState
{
    friend class Evaluator;
 
    explicit EvalState(AsyncIoRoot & aio, Evaluator & ctx);
 
public:
    Evaluator & ctx;
    AsyncIoRoot & aio;
 
    EvalState(const EvalState &) = delete;
    EvalState(EvalState &&) = delete;
    EvalState & operator=(const EvalState &) = delete;
    EvalState & operator=(EvalState &&) = delete;
 
    ~EvalState();

    void evalFile(const SourcePath & path, Value & v);
 
    void resetFileCache();

    void eval(Expr & e, Value & v);

    inline bool evalBool(Env & env, Expr & e);
    inline void evalAttrs(Env & env, Expr & e, Value & v);
    inline void evalList(Env & env, Expr & e, Value & v);

    inline void forceValue(Value & v, const PosIdx pos);
 
    void tryFixupBlackHolePos(Value & v, PosIdx pos);

    void forceValueDeep(Value & v);

    NixInt forceInt(Value & v, const PosIdx pos, std::string_view errorCtx);
    NixFloat forceFloat(Value & v, const PosIdx pos, std::string_view errorCtx);
    bool forceBool(Value & v, const PosIdx pos, std::string_view errorCtx);
 
    void forceAttrs(Value & v, const PosIdx pos, std::string_view errorCtx);
    inline void forceList(Value & v, const PosIdx pos, std::string_view errorCtx);
    void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx);
    std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx);
    std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx);
    std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx);

    [[nodiscard]] StringMap realiseContext(const NixStringContext & context);
 
public:
    bool isDerivation(Value & v);
 
    std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v,
        NixStringContext & context, bool coerceMore = false, bool copyToStore = true);

    BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context,
        std::string_view errorCtx,
        bool coerceMore = false, bool copyToStore = true,
        bool canonicalizePath = true);

    SourcePath coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);

    StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx);

    std::pair<SingleDerivedPath, std::string_view> coerceToSingleDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx);

    SingleDerivedPath coerceToSingleDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx);
 
private:
 
    inline Value * lookupVar(Env * env, const ExprVar & var, bool noEval);
 
    friend struct ExprVar;
    friend struct ExprSet;
    friend struct ExprLet;

    size_t callDepth = 0;
 
public:

    bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx);
 
    bool isFunctor(Value & fun);
 
    // FIXME: use std::span
    void callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos);
 
    void callFunction(Value & fun, Value & arg, Value & vRes, const PosIdx pos)
    {
        Value * args[] = {&arg};
        callFunction(fun, 1, args, vRes, pos);
    }

    void autoCallFunction(Bindings & args, Value & fun, Value & res, PosIdx pos);
 
    void mkPos(Value & v, PosIdx pos);

    void mkOutputString(
        Value & value,
        const SingleDerivedPath::Built & b,
        std::optional<StorePath> optStaticOutputPath,
        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);

    void mkSingleDerivedPathString(
        const SingleDerivedPath & p,
        Value & v);
 
    void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx);
 
private:

    std::string mkOutputStringRaw(
        const SingleDerivedPath::Built & b,
        std::optional<StorePath> optStaticOutputPath,
        const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings);

    std::string mkSingleDerivedPathStringRaw(
        const SingleDerivedPath & p);
};

std::string_view showType(ValueType type, bool withArticle = true);
std::string showType(const Value & v);
 
static constexpr std::string_view corepkgsPrefix{"/__corepkgs__/"};
 
 
}
 
#include "lix/libexpr/eval-inline.hh" // IWYU pragma: keep