~ruther/guix-local

b1fd0ab73425e682a0a1404da6963f47a033bb34 — Eelco Dolstra 10 years ago 7bed5d9 
daemon: Improve the SQLite wrapper API.

In particular, this eliminates a bunch of boilerplate code.

Also integrates these Nix commits:

  80da7a6 Probably fix SQLITE_BUSY errors
  37a337b throwSQLiteError(): Check for SIGINT so we don't loop forever

Co-authored-by: Ludovic Courtès <ludo@gnu.org>
patch browse .tar.gz 
5 files changed, 168 insertions(+), 228 deletions(-)

M nix/libstore/local-store.cc
M nix/libstore/local-store.hh
M nix/libstore/sqlite.cc
M nix/libstore/sqlite.hh
M nix/libstore/store-api.hh

M nix/libstore/local-store.cc => nix/libstore/local-store.cc +68 -175
@@ 329,6 329,7 @@ void LocalStore::openDB(bool create)
    // ensure efficient lookup.
    stmtQueryPathFromHashPart.create(db,
        "select path from ValidPaths where path >= ? limit 1;");
    stmtQueryValidPaths.create(db, "select path from ValidPaths");
}




@@ 523,23 524,16 @@ void LocalStore::checkDerivationOutputs(const Path & drvPath, const Derivation &
}


unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
uint64_t LocalStore::addValidPath(const ValidPathInfo & info, bool checkOutputs)
{
    SQLiteStmtUse use(stmtRegisterValidPath);
    stmtRegisterValidPath.bind(info.path);
    stmtRegisterValidPath.bind("sha256:" + printHash(info.hash));
    stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime);
    if (info.deriver != "")
        stmtRegisterValidPath.bind(info.deriver);
    else
        stmtRegisterValidPath.bind(); // null
    if (info.narSize != 0)
        stmtRegisterValidPath.bind64(info.narSize);
    else
        stmtRegisterValidPath.bind(); // null
    if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE)
        throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path);
    unsigned long long id = sqlite3_last_insert_rowid(db);
    stmtRegisterValidPath.use()
        (info.path)
        ("sha256:" + printHash(info.hash))
        (info.registrationTime == 0 ? time(0) : info.registrationTime)
        (info.deriver, info.deriver != "")
        (info.narSize, info.narSize != 0)
        .exec();
    uint64_t id = sqlite3_last_insert_rowid(db);

    /* If this is a derivation, then store the derivation outputs in
       the database.  This is useful for the garbage collector: it can


@@ 555,13 549,12 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che
           registration above is undone. */
        if (checkOutputs) checkDerivationOutputs(info.path, drv);

        foreach (DerivationOutputs::iterator, i, drv.outputs) {
            SQLiteStmtUse use(stmtAddDerivationOutput);
            stmtAddDerivationOutput.bind(id);
            stmtAddDerivationOutput.bind(i->first);
            stmtAddDerivationOutput.bind(i->second.path);
            if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE)
                throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path);
        for (auto & i : drv.outputs) {
            stmtAddDerivationOutput.use()
                (id)
                (i.first)
                (i.second.path)
                .exec();
        }
    }



@@ 569,24 562,16 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info, bool che
}


void LocalStore::addReference(unsigned long long referrer, unsigned long long reference)
void LocalStore::addReference(uint64_t referrer, uint64_t reference)
{
    SQLiteStmtUse use(stmtAddReference);
    stmtAddReference.bind(referrer);
    stmtAddReference.bind(reference);
    if (sqlite3_step(stmtAddReference) != SQLITE_DONE)
        throwSQLiteError(db, "adding reference to database");
    stmtAddReference.use()(referrer)(reference).exec();
}


void LocalStore::registerFailedPath(const Path & path)
{
    retrySQLite<void>([&]() {
        SQLiteStmtUse use(stmtRegisterFailedPath);
        stmtRegisterFailedPath.bind(path);
        stmtRegisterFailedPath.bind(time(0));
        if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE)
            throwSQLiteError(db, format("registering failed path `%1%'") % path);
        stmtRegisterFailedPath.use()(path)(time(0)).step();
    });
}



@@ 594,12 579,7 @@ void LocalStore::registerFailedPath(const Path & path)
bool LocalStore::hasPathFailed(const Path & path)
{
    return retrySQLite<bool>([&]() {
        SQLiteStmtUse use(stmtHasPathFailed);
        stmtHasPathFailed.bind(path);
        int res = sqlite3_step(stmtHasPathFailed);
        if (res != SQLITE_DONE && res != SQLITE_ROW)
            throwSQLiteError(db, "querying whether path failed");
        return res == SQLITE_ROW;
        return stmtHasPathFailed.use()(path).next();
    });
}



@@ 607,18 587,11 @@ bool LocalStore::hasPathFailed(const Path & path)
PathSet LocalStore::queryFailedPaths()
{
    return retrySQLite<PathSet>([&]() {
        SQLiteStmtUse use(stmtQueryFailedPaths);
        auto useQueryFailedPaths(stmtQueryFailedPaths.use());

        PathSet res;
        int r;
        while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) {
            const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0);
            assert(s);
            res.insert(s);
        }

        if (r != SQLITE_DONE)
            throwSQLiteError(db, "error querying failed paths");
        while (useQueryFailedPaths.next())
            res.insert(useQueryFailedPaths.getStr(0));

        return res;
    });


@@ 630,12 603,8 @@ void LocalStore::clearFailedPaths(const PathSet & paths)
    retrySQLite<void>([&]() {
        SQLiteTxn txn(db);

        foreach (PathSet::const_iterator, i, paths) {
            SQLiteStmtUse use(stmtClearFailedPath);
            stmtClearFailedPath.bind(*i);
            if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE)
                throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i);
        }
        for (auto & path : paths)
            stmtClearFailedPath.use()(path).exec();

        txn.commit();
    });


@@ 666,41 635,28 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
    return retrySQLite<ValidPathInfo>([&]() {

        /* Get the path info. */
        SQLiteStmtUse use1(stmtQueryPathInfo);

        stmtQueryPathInfo.bind(path);
        auto useQueryPathInfo(stmtQueryPathInfo.use()(path));

        int r = sqlite3_step(stmtQueryPathInfo);
        if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
        if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database");
        if (!useQueryPathInfo.next())
            throw Error(format("path `%1%' is not valid") % path);

        info.id = sqlite3_column_int(stmtQueryPathInfo, 0);
        info.id = useQueryPathInfo.getInt(0);

        const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1);
        assert(s);
        info.hash = parseHashField(path, s);
        info.hash = parseHashField(path, useQueryPathInfo.getStr(1));

        info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2);
        info.registrationTime = useQueryPathInfo.getInt(2);

        s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
        auto s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
        if (s) info.deriver = s;

        /* Note that narSize = NULL yields 0. */
        info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4);
        info.narSize = useQueryPathInfo.getInt(4);

        /* Get the references. */
        SQLiteStmtUse use2(stmtQueryReferences);

        stmtQueryReferences.bind(info.id);
        auto useQueryReferences(stmtQueryReferences.use()(info.id));

        while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) {
            s = (const char *) sqlite3_column_text(stmtQueryReferences, 0);
            assert(s);
            info.references.insert(s);
        }

        if (r != SQLITE_DONE)
            throwSQLiteError(db, format("error getting references of `%1%'") % path);
        while (useQueryReferences.next())
            info.references.insert(useQueryReferences.getStr(0));

        return info;
    });


@@ 711,37 667,26 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path)
   narSize field. */
void LocalStore::updatePathInfo(const ValidPathInfo & info)
{
    SQLiteStmtUse use(stmtUpdatePathInfo);
    if (info.narSize != 0)
        stmtUpdatePathInfo.bind64(info.narSize);
    else
        stmtUpdatePathInfo.bind(); // null
    stmtUpdatePathInfo.bind("sha256:" + printHash(info.hash));
    stmtUpdatePathInfo.bind(info.path);
    if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE)
        throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path);
    stmtUpdatePathInfo.use()
        (info.narSize, info.narSize != 0)
        ("sha256:" + printHash(info.hash))
        (info.path)
        .exec();
}


unsigned long long LocalStore::queryValidPathId(const Path & path)
uint64_t LocalStore::queryValidPathId(const Path & path)
{
    SQLiteStmtUse use(stmtQueryPathInfo);
    stmtQueryPathInfo.bind(path);
    int res = sqlite3_step(stmtQueryPathInfo);
    if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0);
    if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
    throwSQLiteError(db, "querying path in database");
    auto use(stmtQueryPathInfo.use()(path));
    if (!use.next())
        throw Error(format("path ‘%1%’ is not valid") % path);
    return use.getInt(0);
}


bool LocalStore::isValidPath_(const Path & path)
{
    SQLiteStmtUse use(stmtQueryPathInfo);
    stmtQueryPathInfo.bind(path);
    int res = sqlite3_step(stmtQueryPathInfo);
    if (res != SQLITE_DONE && res != SQLITE_ROW)
        throwSQLiteError(db, "querying path in database");
    return res == SQLITE_ROW;
    return stmtQueryPathInfo.use()(path).next();
}




@@ 767,20 712,9 @@ PathSet LocalStore::queryValidPaths(const PathSet & paths)
PathSet LocalStore::queryAllValidPaths()
{
    return retrySQLite<PathSet>([&]() {
        SQLiteStmt stmt;
        stmt.create(db, "select path from ValidPaths");

        auto use(stmtQueryValidPaths.use());
        PathSet res;
        int r;
        while ((r = sqlite3_step(stmt)) == SQLITE_ROW) {
            const char * s = (const char *) sqlite3_column_text(stmt, 0);
            assert(s);
            res.insert(s);
        }

        if (r != SQLITE_DONE)
            throwSQLiteError(db, "error getting valid paths");

        while (use.next()) res.insert(use.getStr(0));
        return res;
    });
}


@@ 796,19 730,10 @@ void LocalStore::queryReferences(const Path & path,

void LocalStore::queryReferrers_(const Path & path, PathSet & referrers)
{
    SQLiteStmtUse use(stmtQueryReferrers);

    stmtQueryReferrers.bind(path);

    int r;
    while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) {
        const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0);
        assert(s);
        referrers.insert(s);
    }
    auto useQueryReferrers(stmtQueryReferrers.use()(path));

    if (r != SQLITE_DONE)
        throwSQLiteError(db, format("error getting references of `%1%'") % path);
    while (useQueryReferrers.next())
        referrers.insert(useQueryReferrers.getStr(0));
}




@@ 832,19 757,11 @@ PathSet LocalStore::queryValidDerivers(const Path & path)
    assertStorePath(path);

    return retrySQLite<PathSet>([&]() {
        SQLiteStmtUse use(stmtQueryValidDerivers);
        stmtQueryValidDerivers.bind(path);
        auto useQueryValidDerivers(stmtQueryValidDerivers.use()(path));

        PathSet derivers;
        int r;
        while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) {
            const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1);
            assert(s);
            derivers.insert(s);
        }

        if (r != SQLITE_DONE)
            throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path);
        while (useQueryValidDerivers.next())
            derivers.insert(useQueryValidDerivers.getStr(1));

        return derivers;
    });


@@ 854,19 771,11 @@ PathSet LocalStore::queryValidDerivers(const Path & path)
PathSet LocalStore::queryDerivationOutputs(const Path & path)
{
    return retrySQLite<PathSet>([&]() {
        SQLiteStmtUse use(stmtQueryDerivationOutputs);
        stmtQueryDerivationOutputs.bind(queryValidPathId(path));
        auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));

        PathSet outputs;
        int r;
        while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) {
            const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1);
            assert(s);
            outputs.insert(s);
        }

        if (r != SQLITE_DONE)
            throwSQLiteError(db, format("error getting outputs of `%1%'") % path);
        while (useQueryDerivationOutputs.next())
            outputs.insert(useQueryDerivationOutputs.getStr(1));

        return outputs;
    });


@@ 876,19 785,11 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path)
StringSet LocalStore::queryDerivationOutputNames(const Path & path)
{
    return retrySQLite<StringSet>([&]() {
        SQLiteStmtUse use(stmtQueryDerivationOutputs);
        stmtQueryDerivationOutputs.bind(queryValidPathId(path));
        auto useQueryDerivationOutputs(stmtQueryDerivationOutputs.use()(queryValidPathId(path)));

        StringSet outputNames;
        int r;
        while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) {
            const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 0);
            assert(s);
            outputNames.insert(s);
        }

        if (r != SQLITE_DONE)
            throwSQLiteError(db, format("error getting output names of `%1%'") % path);
        while (useQueryDerivationOutputs.next())
            outputNames.insert(useQueryDerivationOutputs.getStr(0));

        return outputNames;
    });


@@ 902,12 803,9 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
    Path prefix = settings.nixStore + "/" + hashPart;

    return retrySQLite<Path>([&]() -> Path {
        SQLiteStmtUse use(stmtQueryPathFromHashPart);
        stmtQueryPathFromHashPart.bind(prefix);
        auto useQueryPathFromHashPart(stmtQueryPathFromHashPart.use()(prefix));

        int res = sqlite3_step(stmtQueryPathFromHashPart);
        if (res == SQLITE_DONE) return "";
        if (res != SQLITE_ROW) throwSQLiteError(db, "finding path in database");
        if (!useQueryPathFromHashPart.next()) return "";

        const char * s = (const char *) sqlite3_column_text(stmtQueryPathFromHashPart, 0);
        return s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0 ? s : "";


@@ 1154,10 1052,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
            paths.insert(i->path);
        }

        foreach (ValidPathInfos::const_iterator, i, infos) {
            unsigned long long referrer = queryValidPathId(i->path);
            foreach (PathSet::iterator, j, i->references)
                addReference(referrer, queryValidPathId(*j));
        for (auto & i : infos) {
            auto referrer = queryValidPathId(i.path);
            for (auto & j : i.references)
                addReference(referrer, queryValidPathId(j));
        }

        /* Check that the derivation outputs are correct.  We can't do


@@ 1190,12 1088,7 @@ void LocalStore::invalidatePath(const Path & path)

    drvHashes.erase(path);

    SQLiteStmtUse use(stmtInvalidatePath);

    stmtInvalidatePath.bind(path);

    if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE)
        throwSQLiteError(db, format("invalidating path `%1%' in database") % path);
    stmtInvalidatePath.use()(path).exec();

    /* Note that the foreign key constraints on the Refs table take
       care of deleting the references entries for `path'. */


M nix/libstore/local-store.hh => nix/libstore/local-store.hh +4 -3
@@ 207,6 207,7 @@ private:
    SQLiteStmt stmtQueryValidDerivers;
    SQLiteStmt stmtQueryDerivationOutputs;
    SQLiteStmt stmtQueryPathFromHashPart;
    SQLiteStmt stmtQueryValidPaths;

    /* Cache for pathContentsGood(). */
    std::map<Path, bool> pathContentsGoodCache;


@@ 223,11 224,11 @@ private:

    void makeStoreWritable();

    unsigned long long queryValidPathId(const Path & path);
    uint64_t queryValidPathId(const Path & path);

    unsigned long long addValidPath(const ValidPathInfo & info, bool checkOutputs = true);
    uint64_t addValidPath(const ValidPathInfo & info, bool checkOutputs = true);

    void addReference(unsigned long long referrer, unsigned long long reference);
    void addReference(uint64_t referrer, uint64_t reference);

    void appendReferrer(const Path & from, const Path & to, bool lock);



M nix/libstore/sqlite.cc => nix/libstore/sqlite.cc +56 -29
@@ 53,15 53,6 @@ void SQLiteStmt::create(sqlite3 * db, const string & s)
    this->db = db;
}

void SQLiteStmt::reset()
{
    assert(stmt);
    /* Note: sqlite3_reset() returns the error code for the most
       recent call to sqlite3_step().  So ignore it. */
    sqlite3_reset(stmt);
    curArg = 1;
}

SQLiteStmt::~SQLiteStmt()
{
    try {


@@ 72,43 63,79 @@ SQLiteStmt::~SQLiteStmt()
    }
}

void SQLiteStmt::bind(const string & value)
SQLiteStmt::Use::Use(SQLiteStmt & stmt)
    : stmt(stmt)
{
    assert(stmt.stmt);
    /* Note: sqlite3_reset() returns the error code for the most
       recent call to sqlite3_step().  So ignore it. */
    sqlite3_reset(stmt);
}

SQLiteStmt::Use::~Use()
{
    if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
        throwSQLiteError(db, "binding argument");
    sqlite3_reset(stmt);
}

void SQLiteStmt::bind(int value)
SQLiteStmt::Use & SQLiteStmt::Use::operator () (const std::string & value, bool notNull)
{
    if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK)
        throwSQLiteError(db, "binding argument");
    if (notNull) {
        if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
            throwSQLiteError(stmt.db, "binding argument");
    } else
        bind();
    return *this;
}

void SQLiteStmt::bind64(long long value)
SQLiteStmt::Use & SQLiteStmt::Use::operator () (int64_t value, bool notNull)
{
    if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
        throwSQLiteError(db, "binding argument");
    if (notNull) {
        if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK)
            throwSQLiteError(stmt.db, "binding argument");
    } else
        bind();
    return *this;
}

void SQLiteStmt::bind()
SQLiteStmt::Use & SQLiteStmt::Use::bind()
{
    if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
        throwSQLiteError(db, "binding argument");
        throwSQLiteError(stmt.db, "binding argument");
    return *this;
}

SQLiteStmtUse::SQLiteStmtUse(SQLiteStmt & stmt)
    : stmt(stmt)
int SQLiteStmt::Use::step()
{
    stmt.reset();
    return sqlite3_step(stmt);
}

SQLiteStmtUse::~SQLiteStmtUse()
void SQLiteStmt::Use::exec()
{
    try {
        stmt.reset();
    } catch (...) {
        ignoreException();
    }
    int r = step();
    assert(r != SQLITE_ROW);
    if (r != SQLITE_DONE)
        throwSQLiteError(stmt.db, "executing SQLite statement");
}

bool SQLiteStmt::Use::next()
{
    int r = step();
    if (r != SQLITE_DONE && r != SQLITE_ROW)
        throwSQLiteError(stmt.db, "executing SQLite query");
    return r == SQLITE_ROW;
}

std::string SQLiteStmt::Use::getStr(int col)
{
    auto s = (const char *) sqlite3_column_text(stmt, col);
    assert(s);
    return s;
}

int64_t SQLiteStmt::Use::getInt(int col)
{
    // FIXME: detect nulls?
    return sqlite3_column_int64(stmt, col);
}

SQLiteTxn::SQLiteTxn(sqlite3 * db)


M nix/libstore/sqlite.hh => nix/libstore/sqlite.hh +38 -19
@@ 22,29 22,48 @@ struct SQLite
/* RAII wrapper to create and destroy SQLite prepared statements. */
struct SQLiteStmt
{
    sqlite3 * db;
    sqlite3_stmt * stmt;
    unsigned int curArg;
    SQLiteStmt() { stmt = 0; }
    sqlite3 * db = 0;
    sqlite3_stmt * stmt = 0;
    SQLiteStmt() { }
    void create(sqlite3 * db, const std::string & s);
    void reset();
    ~SQLiteStmt();
    operator sqlite3_stmt * () { return stmt; }
    void bind(const std::string & value);
    void bind(int value);
    void bind64(long long value);
    void bind();
};

/* Helper class to ensure that prepared statements are reset when
   leaving the scope that uses them.  Unfinished prepared statements
   prevent transactions from being aborted, and can cause locks to be
   kept when they should be released. */
struct SQLiteStmtUse
{
    SQLiteStmt & stmt;
    SQLiteStmtUse(SQLiteStmt & stmt);
    ~SQLiteStmtUse();
    /* Helper for binding / executing statements. */
    class Use
    {
        friend struct SQLiteStmt;
    private:
        SQLiteStmt & stmt;
        unsigned int curArg = 1;
        Use(SQLiteStmt & stmt);

    public:

        ~Use();

        /* Bind the next parameter. */
        Use & operator () (const std::string & value, bool notNull = true);
        Use & operator () (int64_t value, bool notNull = true);
        Use & bind(); // null

        int step();

        /* Execute a statement that does not return rows. */
        void exec();

        /* For statements that return 0 or more rows. Returns true iff
           a row is available. */
        bool next();

        std::string getStr(int col);
        int64_t getInt(int col);
    };

    Use use()
    {
        return Use(*this);
    }
};

/* RAII helper that ensures transactions are aborted unless explicitly


M nix/libstore/store-api.hh => nix/libstore/store-api.hh +2 -2
@@ 89,8 89,8 @@ struct ValidPathInfo
    Hash hash;
    PathSet references;
    time_t registrationTime = 0;
    unsigned long long narSize = 0; // 0 = unknown
    unsigned long long id; // internal use only
    uint64_t narSize = 0; // 0 = unknown
    uint64_t id; // internal use only

    bool operator == (const ValidPathInfo & i) const
    {