worker.hh

Go to the documentation of this file.

#pragma once
 
#include "lix/libutil/async.hh"
#include "lix/libutil/async-semaphore.hh"
#include "lix/libutil/concepts.hh"
#include "lix/libutil/notifying-counter.hh"
#include "lix/libutil/result.hh"
#include "lix/libutil/types.hh"
#include "lix/libstore/lock.hh"
#include "lix/libstore/store-api.hh"
#include "lix/libstore/build/goal.hh"
#include "lix/libstore/realisation.hh"
 
#include <future>
#include <kj/async-io.h>
#include <thread>
 
namespace nix {
 
/* Forward definition. */
struct DerivationGoal;
struct PathSubstitutionGoal;
class DrvOutputSubstitutionGoal;
class LocalStore;
 
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
 
/* Forward definition. */
struct HookInstance;
 
class GoalFactory
{
public:
    virtual std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>>
    makeDerivationGoal(
        const StorePath & drvPath, const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal
    ) = 0;
    virtual kj::Promise<
        Result<std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>>>>
    makeBasicDerivationGoal(
        const StorePath & drvPath,
        const BasicDerivation & drv,
        const OutputsSpec & wantedOutputs,
        BuildMode buildMode = bmNormal
    ) = 0;

    virtual std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>>
    makePathSubstitutionGoal(
        const StorePath & storePath,
        RepairFlag repair = NoRepair,
        std::optional<ContentAddress> ca = std::nullopt
    ) = 0;
    virtual std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>>
    makeDrvOutputSubstitutionGoal(
        const DrvOutput & id,
        RepairFlag repair = NoRepair,
        std::optional<ContentAddress> ca = std::nullopt
    ) = 0;

    virtual std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>>
    makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) = 0;
};
 
// elaborate hoax to let goals access factory methods while hiding them from the public
class WorkerBase : protected GoalFactory
{
    friend struct DerivationGoal;
    friend struct PathSubstitutionGoal;
    friend class DrvOutputSubstitutionGoal;
 
protected:
    GoalFactory & goalFactory() { return *this; }
};

class Worker : public WorkerBase
{
public:
    using Targets = std::vector<std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>>>;
    struct Results {
        std::map<size_t, Goal::WorkResult> goals;

        unsigned int failingExitStatus;
    };
 
private:
 
    bool running = false;
 
    template<typename G>
    struct CachedGoal
    {
        std::shared_ptr<G> goal;
        kj::ForkedPromise<Result<Goal::WorkResult>> promise{nullptr};
    };
    std::map<StorePath, CachedGoal<DerivationGoal>> derivationGoals;
    std::map<StorePath, CachedGoal<PathSubstitutionGoal>> substitutionGoals;
    std::map<DrvOutput, CachedGoal<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;

    std::map<StorePath, bool> pathContentsGoodCache;

    bool permanentFailure = false;

    bool timedOut = false;

    bool hashMismatch = false;

    bool checkMismatch = false;

    kj::Promise<Result<Results>> updateStatistics();
 
    AsyncSemaphore statisticsUpdateSignal{1};
    std::optional<AsyncSemaphore::Token> statisticsUpdateInhibitor;

    void updateStatisticsLater()
    {
        statisticsUpdateInhibitor = {};
    }
 
    kj::Promise<Result<Results>> runImpl(Targets topGoals);
    kj::Promise<Result<Results>> boopGC(LocalStore & localStore);
 
public:
 
    const Activity act;
    const Activity actDerivations;
    const Activity actSubstitutions;
 
    Store & store;
    Store & evalStore;
    AsyncSemaphore substitutions, localBuilds;
    std::optional<Path> buildDirOverride;
 
private:
    kj::TaskSet children;
 
public:
    struct HookState {
        std::unique_ptr<HookInstance> instance;

        bool available = true;
    };
 
    HookState hook;
 
    NotifyingCounter<uint64_t> expectedBuilds{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> doneBuilds{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> failedBuilds{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> runningBuilds{[this] { updateStatisticsLater(); }};
 
    NotifyingCounter<uint64_t> expectedSubstitutions{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> doneSubstitutions{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> failedSubstitutions{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> runningSubstitutions{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> expectedDownloadSize{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> doneDownloadSize{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> expectedNarSize{[this] { updateStatisticsLater(); }};
    NotifyingCounter<uint64_t> doneNarSize{[this] { updateStatisticsLater(); }};
 
private:
    Worker(Store & store, Store & evalStore);
    ~Worker();


    template<typename ID, std::derived_from<Goal> G>
    std::pair<std::shared_ptr<G>, kj::Promise<Result<Goal::WorkResult>>> makeGoalCommon(
        std::map<ID, CachedGoal<G>> & map,
        const ID & key,
        InvocableR<std::unique_ptr<G>> auto create,
        InvocableR<bool, G &> auto modify
    );
    std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>> makeDerivationGoal(
        const StorePath & drvPath,
        const OutputsSpec & wantedOutputs, BuildMode buildMode = bmNormal) override;
    kj::Promise<
        Result<std::pair<std::shared_ptr<DerivationGoal>, kj::Promise<Result<Goal::WorkResult>>>>>
    makeBasicDerivationGoal(
        const StorePath & drvPath,
        const BasicDerivation & drv,
        const OutputsSpec & wantedOutputs,
        BuildMode buildMode = bmNormal
    ) override;

    std::pair<std::shared_ptr<PathSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>>
    makePathSubstitutionGoal(
        const StorePath & storePath,
        RepairFlag repair = NoRepair,
        std::optional<ContentAddress> ca = std::nullopt
    ) override;
    std::pair<std::shared_ptr<DrvOutputSubstitutionGoal>, kj::Promise<Result<Goal::WorkResult>>>
    makeDrvOutputSubstitutionGoal(
        const DrvOutput & id,
        RepairFlag repair = NoRepair,
        std::optional<ContentAddress> ca = std::nullopt
    ) override;

    std::pair<GoalPtr, kj::Promise<Result<Goal::WorkResult>>>
    makeGoal(const DerivedPath & req, BuildMode buildMode = bmNormal) override;
 
    kj::Promise<Result<Results>> run(Targets topGoals);
 
public:
    kj::Promise<Result<Results>> run(std::function<kj::Promise<Result<Targets>>(GoalFactory &)> req)
    try {
        co_return co_await run(TRY_AWAIT(req(goalFactory())));
    } catch (...) {
        co_return result::current_exception();
    }
 
    kj::Promise<Result<Results>> run(std::function<Targets(GoalFactory &)> req)
    try {
        return run(req(goalFactory()));
    } catch (...) {
        return {result::current_exception()};
    }

    kj::Promise<Result<bool>> pathContentsGood(const StorePath & path);
 
    void markContentsGood(const StorePath & path);
 
    template<typename MkGoals>
    friend kj::Promise<Result<Results>>
    processGoals(Store & store, Store & evalStore, MkGoals && mkGoals) noexcept;
};
 
template<typename MkGoals>
kj::Promise<Result<Worker::Results>>
processGoals(Store & store, Store & evalStore, MkGoals && mkGoals) noexcept
try {
    co_return co_await Worker(store, evalStore).run(std::forward<MkGoals>(mkGoals));
} catch (...) {
    co_return result::current_exception();
}
}