Skip to content

Commit 8df800c

Browse files
Ericson2314Ericson2314
authored
Merge pull request #7 from obsidiansystems/git+validPathInfo-ca-proper-datatype
valid path info ca proper datatype
2 parents 8bbe10e + 010013e commit 8df800c

24 files changed

+243
-156
lines changed

‎src/libexpr/get-drvs.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
#include "get-drvs.hh"
22
#include "util.hh"
33
#include "eval-inline.hh"
4-
#include "derivations.hh"
4+
#include "store-api.hh"
55

66
#include <cstring>
77
#include <regex>

‎src/libexpr/primops/context.cc

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
#include "primops.hh"
22
#include "eval-inline.hh"
3-
#include "derivations.hh"
3+
#include "store-api.hh"
44

55
namespace nix {
66

‎src/libfetchers/tarball.cc

+4-1
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,10 @@ DownloadFileResult downloadFile(
7070
ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
7171
info.narHash = hashString(HashType::SHA256, *sink.s);
7272
info.narSize = sink.s->size();
73-
info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
73+
info.ca = FileSystemHash {
74+
FileIngestionMethod::Flat,
75+
hash,
76+
};
7477
auto source = StringSource { *sink.s };
7578
store->addToStore(info, source, NoRepair, NoCheckSigs);
7679
storePath = std::move(info.path);

‎src/libstore/build.cc

+6-3
Original file line numberDiff line numberDiff line change
@@ -3708,7 +3708,7 @@ void DerivationGoal::registerOutputs()
37083708
/* Check that fixed-output derivations produced the right
37093709
outputs (i.e., the content hash should match the specified
37103710
hash). */
3711-
std::string ca;
3711+
std::optional<ContentAddress> ca;
37123712

37133713
if (fixedOutput) {
37143714

@@ -3757,7 +3757,7 @@ void DerivationGoal::registerOutputs()
37573757
else
37583758
assert(worker.store.parseStorePath(path) == dest);
37593759

3760-
ca = makeFixedOutputCA(i.second.hash->method, h2);
3760+
ca = FileSystemHash { i.second.hash->method, h2 };
37613761
}
37623762

37633763
/* Get rid of all weird permissions. This also checks that
@@ -3830,7 +3830,10 @@ void DerivationGoal::registerOutputs()
38303830
info.ca = ca;
38313831
worker.store.signPathInfo(info);
38323832

3833-
if (!info.references.empty()) info.ca.clear();
3833+
if (!info.references.empty()) {
3834+
// FIXME don't we have an experimental feature for fixed output with references?
3835+
info.ca = {};
3836+
}
38343837

38353838
infos.emplace(i.first, std::move(info));
38363839
}

‎src/libstore/content-address.cc

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#include "content-address.hh"
2+
3+
namespace nix {
4+
5+
std::string FileSystemHash::printMethodAlgo() const {
6+
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
7+
}
8+
9+
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
10+
switch (m) {
11+
case FileIngestionMethod::Flat:
12+
return "";
13+
case FileIngestionMethod::Recursive:
14+
return "r:";
15+
case FileIngestionMethod::Git:
16+
return "git:";
17+
}
18+
abort();
19+
}
20+
21+
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash)
22+
{
23+
return "fixed:"
24+
+ makeFileIngestionPrefix(method)
25+
+ hash.to_string();
26+
}
27+
28+
// FIXME Put this somewhere?
29+
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
30+
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
31+
32+
std::string renderContentAddress(ContentAddress ca) {
33+
return std::visit(overloaded {
34+
[](TextHash th) {
35+
return "text:" + th.hash.to_string();
36+
},
37+
[](FileSystemHash fsh) {
38+
return makeFixedOutputCA(fsh.method, fsh.hash);
39+
}
40+
}, ca);
41+
}
42+
43+
ContentAddress parseContentAddress(std::string_view rawCa) {
44+
auto prefixSeparator = rawCa.find(':');
45+
if (prefixSeparator != string::npos) {
46+
auto prefix = string(rawCa, 0, prefixSeparator);
47+
if (prefix == "text") {
48+
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
49+
Hash hash = Hash(string(hashTypeAndHash));
50+
if (*hash.type != HashType::SHA256) {
51+
throw Error("parseContentAddress: the text hash should have type SHA256");
52+
}
53+
return TextHash { hash };
54+
} else if (prefix == "fixed") {
55+
// This has to be an inverse of makeFixedOutputCA
56+
auto methodAndHash = rawCa.substr(prefixSeparator+1, string::npos);
57+
if (methodAndHash.substr(0, 2) == "r:") {
58+
std::string_view hashRaw = methodAndHash.substr(2, string::npos);
59+
return FileSystemHash { FileIngestionMethod::Recursive, Hash(string(hashRaw)) };
60+
} else if (methodAndHash.substr(0, 4) == "git:") {
61+
std::string_view hashRaw = methodAndHash.substr(4, string::npos);
62+
return FileSystemHash { FileIngestionMethod::Git, Hash(string(hashRaw)) };
63+
} else {
64+
std::string_view hashRaw = methodAndHash;
65+
return FileSystemHash { FileIngestionMethod::Flat, Hash(string(hashRaw)) };
66+
}
67+
} else {
68+
throw Error("parseContentAddress: format not recognized; has to be text or fixed");
69+
}
70+
} else {
71+
throw Error("Not a content address because it lacks an appropriate prefix");
72+
}
73+
};
74+
75+
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt) {
76+
return rawCaOpt == "" ? std::optional<ContentAddress> {} : parseContentAddress(rawCaOpt);
77+
};
78+
79+
std::string renderContentAddress(std::optional<ContentAddress> ca) {
80+
return ca ? renderContentAddress(*ca) : "";
81+
}
82+
83+
}

‎src/libstore/content-address.hh

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#pragma once
2+
3+
#include <variant>
4+
#include "hash.hh"
5+
6+
namespace nix {
7+
8+
enum struct FileIngestionMethod : uint8_t {
9+
Flat,
10+
Recursive,
11+
Git,
12+
};
13+
14+
15+
struct TextHash {
16+
Hash hash;
17+
TextHash(const TextHash &) = default;
18+
TextHash(TextHash &&) = default;
19+
TextHash & operator = (const TextHash &) = default;
20+
};
21+
22+
/// Pair of a hash, and how the file system was ingested
23+
struct FileSystemHash {
24+
FileIngestionMethod method;
25+
Hash hash;
26+
FileSystemHash(FileIngestionMethod method, Hash hash)
27+
: method(std::move(method))
28+
, hash(std::move(hash))
29+
{ }
30+
FileSystemHash(const FileSystemHash &) = default;
31+
FileSystemHash(FileSystemHash &&) = default;
32+
FileSystemHash & operator = (const FileSystemHash &) = default;
33+
std::string printMethodAlgo() const;
34+
};
35+
36+
/*
37+
We've accumulated several types of content-addressed paths over the years;
38+
fixed-output derivations support multiple hash algorithms and serialisation
39+
methods (flat file vs NAR). Thus, ‘ca’ has one of the following forms:
40+
41+
* ‘text:sha256:<sha256 hash of file contents>’: For paths
42+
computed by makeTextPath() / addTextToStore().
43+
44+
* ‘fixed:<r?>:<ht>:<h>’: For paths computed by
45+
makeFixedOutputPath() / addToStore().
46+
*/
47+
typedef std::variant<
48+
TextHash, // for paths computed by makeTextPath() / addTextToStore
49+
FileSystemHash // for path computed by makeFixedOutputPath
50+
> ContentAddress;
51+
52+
/* Compute the prefix to the hash algorithm which indicates how the files were
53+
ingested. */
54+
std::string makeFileIngestionPrefix(const FileIngestionMethod m);
55+
56+
/* Compute the content-addressability assertion (ValidPathInfo::ca)
57+
for paths created by makeFixedOutputPath() / addToStore(). */
58+
std::string makeFixedOutputCA(FileIngestionMethod method, const Hash & hash);
59+
60+
std::string renderContentAddress(ContentAddress ca);
61+
62+
std::string renderContentAddress(std::optional<ContentAddress> ca);
63+
64+
ContentAddress parseContentAddress(std::string_view rawCa);
65+
66+
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
67+
68+
}

‎src/libstore/daemon.cc

+3-2
Original file line numberDiff line numberDiff line change
@@ -663,7 +663,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
663663
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
664664
to << info->ultimate
665665
<< info->sigs
666-
<< info->ca;
666+
<< renderContentAddress(info->ca);
667667
}
668668
} else {
669669
assert(GET_PROTOCOL_MINOR(clientVersion) >= 17);
@@ -721,7 +721,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
721721
info.references = readStorePaths<StorePathSet>(*store, from);
722722
from >> info.registrationTime >> info.narSize >> info.ultimate;
723723
info.sigs = readStrings<StringSet>(from);
724-
from >> info.ca >> repair >> dontCheckSigs;
724+
info.ca = parseContentAddressOpt(readString(from));
725+
from >> repair >> dontCheckSigs;
725726
if (!trusted && dontCheckSigs)
726727
dontCheckSigs = false;
727728
if (!trusted)

‎src/libstore/derivations.cc

-5
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,6 @@
88

99
namespace nix {
1010

11-
std::string FileSystemHash::printMethodAlgo() const {
12-
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
13-
}
14-
15-
1611
BasicDerivation::BasicDerivation(const BasicDerivation & other)
1712
: platform(other.platform)
1813
, builder(other.builder)

‎src/libstore/derivations.hh

+3-15
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,9 @@
11
#pragma once
22

3+
#include "path.hh"
34
#include "types.hh"
45
#include "hash.hh"
5-
#include "store-api.hh"
6+
#include "content-address.hh"
67

78
#include <map>
89

@@ -12,20 +13,6 @@ namespace nix {
1213

1314
/* Abstract syntax of derivations. */
1415

15-
/// Pair of a hash, and how the file system was ingested
16-
struct FileSystemHash {
17-
FileIngestionMethod method;
18-
Hash hash;
19-
FileSystemHash(FileIngestionMethod method, Hash hash)
20-
: method(std::move(method))
21-
, hash(std::move(hash))
22-
{ }
23-
FileSystemHash(const FileSystemHash &) = default;
24-
FileSystemHash(FileSystemHash &&) = default;
25-
FileSystemHash & operator = (const FileSystemHash &) = default;
26-
std::string printMethodAlgo() const;
27-
};
28-
2916
struct DerivationOutput
3017
{
3118
StorePath path;
@@ -90,6 +77,7 @@ struct Derivation : BasicDerivation
9077

9178
class Store;
9279

80+
enum RepairFlag : bool { NoRepair = false, Repair = true };
9381

9482
/* Write a derivation to the Nix store, and return its path. */
9583
StorePath writeDerivation(ref<Store> store,

‎src/libstore/legacy-ssh-store.cc

+2-2
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ struct LegacySSHStore : public Store
114114
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
115115
auto s = readString(conn->from);
116116
info->narHash = s.empty() ? Hash() : Hash(s);
117-
conn->from >> info->ca;
117+
info->ca = parseContentAddressOpt(readString(conn->from));
118118
info->sigs = readStrings<StringSet>(conn->from);
119119
}
120120

@@ -146,7 +146,7 @@ struct LegacySSHStore : public Store
146146
<< info.narSize
147147
<< info.ultimate
148148
<< info.sigs
149-
<< info.ca;
149+
<< renderContentAddress(info.ca);
150150
try {
151151
copyNAR(source, conn->to);
152152
} catch (...) {

‎src/libstore/local-store.cc

+13-12
Original file line numberDiff line numberDiff line change
@@ -577,7 +577,7 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
577577
uint64_t LocalStore::addValidPath(State & state,
578578
const ValidPathInfo & info, bool checkOutputs)
579579
{
580-
if (info.ca != "" && !info.isContentAddressed(*this))
580+
if (info.ca.has_value() && !info.isContentAddressed(*this))
581581
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
582582
printStorePath(info.path));
583583

@@ -589,7 +589,7 @@ uint64_t LocalStore::addValidPath(State & state,
589589
(info.narSize, info.narSize != 0)
590590
(info.ultimate ? 1 : 0, info.ultimate)
591591
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
592-
(info.ca, !info.ca.empty())
592+
(renderContentAddress(info.ca), (bool) info.ca)
593593
.exec();
594594
uint64_t id = sqlite3_last_insert_rowid(state.db);
595595

@@ -663,7 +663,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
663663
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
664664

665665
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
666-
if (s) info->ca = s;
666+
if (s) info->ca = parseContentAddressOpt(s);
667667

668668
/* Get the references. */
669669
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
@@ -686,7 +686,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
686686
(info.narHash.to_string(Base::Base16))
687687
(info.ultimate ? 1 : 0, info.ultimate)
688688
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
689-
(info.ca, !info.ca.empty())
689+
(renderContentAddress(info.ca), (bool) info.ca)
690690
(printStorePath(info.path))
691691
.exec();
692692
}
@@ -1000,15 +1000,15 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
10001000

10011001
deletePath(realPath);
10021002

1003-
if (info.ca != "" &&
1004-
!((hasPrefix(info.ca, "text:") && !info.references.count(info.path))
1005-
|| info.references.empty()))
1003+
// text hashing has long been allowed to have non-self-references because it is used for drv files.
1004+
bool refersToSelf = info.references.count(info.path) > 0;
1005+
if (info.ca.has_value() && !info.references.empty() && !(std::holds_alternative<TextHash>(*info.ca) && !refersToSelf))
10061006
settings.requireExperimentalFeature("ca-references");
10071007

10081008
/* While restoring the path from the NAR, compute the hash
10091009
of the NAR. */
10101010
std::unique_ptr<AbstractHashSink> hashSink;
1011-
if (info.ca == "" || !info.references.count(info.path))
1011+
if (!info.ca.has_value() || !info.references.count(info.path))
10121012
hashSink = std::make_unique<HashSink>(HashType::SHA256);
10131013
else
10141014
hashSink = std::make_unique<HashModuloSink>(HashType::SHA256, storePathToHash(printStorePath(info.path)));
@@ -1019,7 +1019,8 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
10191019
return n;
10201020
});
10211021

1022-
if (hasPrefix(info.ca, "fixed:git:"))
1022+
auto p = info.ca ? std::get_if<FileSystemHash>(&*info.ca) : NULL;
1023+
if (p && p->method == FileIngestionMethod::Git)
10231024
restoreGit(realPath, wrapperSource, realStoreDir, storeDir);
10241025
else
10251026
restorePath(realPath, wrapperSource);
@@ -1110,7 +1111,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
11101111
ValidPathInfo info(dstPath.clone());
11111112
info.narHash = hash.first;
11121113
info.narSize = hash.second;
1113-
info.ca = makeFixedOutputCA(method, h);
1114+
info.ca = FileSystemHash { method, h };
11141115
registerValidPath(info);
11151116
}
11161117

@@ -1195,7 +1196,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
11951196
info.narHash = narHash;
11961197
info.narSize = sink.s->size();
11971198
info.references = cloneStorePathSet(references);
1198-
info.ca = "text:" + hash.to_string();
1199+
info.ca = TextHash { .hash = hash };
11991200
registerValidPath(info);
12001201
}
12011202

@@ -1303,7 +1304,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
13031304
printMsg(Verbosity::Talkative, "checking contents of '%s'", printStorePath(i));
13041305

13051306
std::unique_ptr<AbstractHashSink> hashSink;
1306-
if (info->ca == "" || !info->references.count(info->path))
1307+
if (!info->ca || !info->references.count(info->path))
13071308
hashSink = std::make_unique<HashSink>(*info->narHash.type);
13081309
else
13091310
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, storePathToHash(printStorePath(info->path)));

0 commit comments

Comments
 (0)