Skip to content

Commit

Permalink
Fixes #74
Browse files Browse the repository at this point in the history
  • Loading branch information
@John-LittleBearLabs
John-LittleBearLabs committed Nov 15, 2023
1 parent 4ef85ac commit b40c99a
Show file tree
Hide file tree
Showing 59 changed files with 711 additions and 2,280 deletions.
49 changes: 39 additions & 10 deletions cmake/patch.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/usr/bin/env python3

import sys
from os import listdir, remove
from os.path import dirname, join, realpath, splitext
from subprocess import check_call, check_output
Expand Down Expand Up @@ -150,7 +150,7 @@ def electron_version(self, branch='main'):
def unavailable(self):
avail = list(map(as_int, self.available()))
version_set = {}
fuzz = 59877
fuzz = 59878
def check(version, version_set, s):
i = as_int(version)
by = (fuzz,0)
Expand Down Expand Up @@ -182,10 +182,19 @@ def check(version, version_set, s):
return map(lambda x: x[1:], result)

def out_of_date(self, p):
with open(f'{self.pdir}/{p}.patch') as f:
file_path = f'{self.pdir}/{p}.patch'
with open(file_path) as f:
lines = f.readlines()
fl = Patcher.file_lines(lines, 'chrome/browser/flag-metadata.json')
return '+ "name": "enable-ipfs",\n' in fl
if not Patcher.has_file_line(lines, 'chrome/browser/flag-metadata.json', '+ "name": "enable-ipfs",'):
print(p, 'does not have enable-ipfs in flag-metadata.json', file_path, file=sys.stderr)
return True
return False

@staticmethod
def has_file_line(lines: list[str], path: str, line: str):
fl = Patcher.file_lines(lines, path)
return (line + '\n') in fl


@staticmethod
def file_lines(lines: list[str], path):
Expand All @@ -194,12 +203,16 @@ def file_lines(lines: list[str], path):
# print('Did not find',start,'in',lines)
return []
i = lines.index(start) + 1
#print(start,'found at',i)
for j in range(i, i + 9999):
if len(lines) == j or lines[j].startswith('diff --git'):
return lines[i:j]
return []

def list_ood(self, list: list[str], sense: bool):
for p in list:
if self.out_of_date(p) == sense:
print(p)


if __name__ == '__main__':
if argv[1] == 'apply':
Expand All @@ -221,10 +234,26 @@ def file_lines(lines: list[str], path):
elif argv[1] == 'old':
pr = Patcher('/mnt/big/lbl/code/chromium/src', 'git', 'Debug')
if len(argv) > 2:
for p in argv[2:]:
print(p, pr.out_of_date(p))
pr.list_ood(argv[2:], True)
else:
pr.list_ood(list(pr.available()), True)
elif argv[1] == 'new':
pr = Patcher('/mnt/big/lbl/code/chromium/src', 'git', 'Debug')
if len(argv) > 2:
pr.list_ood(argv[2:], False)
else:
for p in pr.available():
print(p, pr.out_of_date(p))
pr.list_ood(list(pr.available()), False)
elif argv[1] == 'git':
pr = Patcher(realpath(join(dirname(__file__), '..')), 'git', 'Debug')
pre = '?? component/patches/'
suf = 'patch'
for line in pr.git(['status','--porcelain']).splitlines():
if line.startswith(pre) and line.endswith(suf):
end = len(line) - len(suf) - 1
pch = line[len(pre):end]
if pr.out_of_date(pch):
exit(9)
else:
pr.git(['add', line[3:]])
else:
Patcher(*argv[1:]).create_patch_file()
15 changes: 14 additions & 1 deletion component/block_http_request.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,22 @@ void Self::send(raw_ptr<network::mojom::URLLoaderFactory> loader_factory) {
void Self::OnResponse(std::shared_ptr<Self>,
std::unique_ptr<std::string> body) {
DCHECK(loader_);
int status = loader_->NetError() ? 500 : 200;
int status;
switch (loader_->NetError()) {
case net::Error::OK:
status = 200;
break;
case net::Error::ERR_TIMED_OUT:
status = 408;
break;
default:
status = 500;
}
ContextApi::HeaderAccess header_accessor = [](auto) { return std::string{}; };
auto sz = body ? body->size() : 0UL;
if (loader_->NetError()) {
LOG(WARNING) << "NetErr " << loader_->NetError() << " for " << inf_.url;
}
VLOG(1) << "Handling HTTP response body of size " << sz << " with NetErr "
<< loader_->NetError() << " for " << inf_.url;
auto const* head = loader_->ResponseInfo();
Expand Down
6 changes: 4 additions & 2 deletions component/cache_requestor.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ void Self::RequestByCid(std::string cid,
StartFetch(task, static_cast<net::RequestPriority>(p));
}
void Self::StartFetch(Task& task, net::RequestPriority priority) {
/*
if (pending_) {
Start();
task.Fail();
Expand All @@ -79,6 +80,7 @@ void Self::StartFetch(Task& task, net::RequestPriority priority) {
if (res.net_error() != net::ERR_IO_PENDING) {
OnOpen(task, std::move(res));
}
*/
}

namespace {
Expand All @@ -92,7 +94,7 @@ std::shared_ptr<dc::Entry> GetEntry(dc::EntryResult& result) {
return {e, deleter};
}
} // namespace

/*
void Self::OnOpen(Task task, dc::EntryResult res) {
VLOG(2) << "OnOpen(" << res.net_error() << ")";
if (res.net_error() != net::OK) {
Expand Down Expand Up @@ -146,7 +148,7 @@ void Self::OnBodyRead(Task task, int code) {
task.hit(task.body, task.header);
}
}

*/
void Self::Store(std::string cid, std::string headers, std::string body) {
VLOG(2) << "Store(" << name() << ',' << cid << ',' << headers.size() << ','
<< body.size() << ')';
Expand Down
190 changes: 1 addition & 189 deletions component/chromium_ipfs_context.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,183 +87,6 @@ void Self::Discover(std::function<void(std::vector<std::string>)> cb) {
"https://orchestrator.strn.pl/nodes/nearby";
discovery_loader->DownloadToString(loader_factory_, std::move(bound), MB);
}

auto Self::InitiateGatewayRequest(BusyGateway assigned)
-> std::shared_ptr<GatewayRequest> {
auto url = assigned.url();

GOOGLE_DCHECK_GT(url.size(), 0U);
auto url_suffix = assigned.task();
auto req = std::make_unique<network::ResourceRequest>();
req->url = GURL{url};
req->priority = net::HIGHEST; // TODO
if (!assigned.accept().empty()) {
req->headers.SetHeader("Accept", assigned.accept());
}
auto out = std::make_shared<GatewayUrlLoader>(std::move(assigned));
GOOGLE_DCHECK_GT(out->gateway->url_prefix().size(), 0U);
out->loader = network::SimpleURLLoader::Create(std::move(req),
kTrafficAnnotation, FROM_HERE);
if (url.find("format=ipns-record") == std::string::npos) {
out->loader->SetTimeoutDuration(base::Seconds(32));
} else {
VLOG(1) << "Doing an IPNS record query, so giving it a long timeout.";
out->loader->SetTimeoutDuration(base::Seconds(256));
}
auto start_time = base::TimeTicks::Now();
// out->listener = listener;
auto cb = base::BindOnce(&Self::OnResponse, base::Unretained(this),
shared_from_this(), out, start_time);
DCHECK(loader_factory_);
// TODO - proper requesting with full features (SetPriority, etc.).
out->loader->DownloadToString(loader_factory_, std::move(cb), 2UL * MB);
return out;
}
void Self::OnResponse(std::shared_ptr<ContextApi> api,
std::shared_ptr<GatewayUrlLoader> req,
base::TimeTicks start_time,
std::unique_ptr<std::string> body) {
auto sz = body ? body->size() : 0UL;
LOG(INFO) << "OnResponse(...," << start_time << ", " << sz << "B )";
DCHECK(req);
auto task = req->task();
if (task.empty()) {
LOG(ERROR) << "Got a response for an empty task!";
return;
}
auto url = req->url();
// LOG(INFO) << "Got a response for " << task << " via " << url;
auto& bg = req->gateway;
auto& ldr = req->loader;
// auto listener = req->listener;
if (ProcessResponse(bg, ldr.get(), body.get(), start_time)) {
LOG(INFO) << url << " success.";
bg.Success(state_->gateways(), shared_from_this());
} else {
LOG(INFO) << url << " failure.";
bg.Failure(state_->gateways(), shared_from_this());
}
VLOG(1) << "Recheck for more activity.";
state_->storage().CheckListening();
state_->scheduler().IssueRequests(api);
}

bool Self::ProcessResponse(BusyGateway& gw,
network::SimpleURLLoader* ldr,
std::string* body,
base::TimeTicks start_time) {
auto end_time = base::TimeTicks::Now();
if (!gw) {
LOG(ERROR) << "No gateway.";
return false;
}
VLOG(2) << "ProcessResponse(" << gw.url() << ')';
if (!ldr) {
LOG(ERROR) << "No loader for processing " << gw.url();
return false;
}
if (!body) {
LOG(INFO) << "ProcessResponse(" << gw.url()
<< ") Null body - presumably http error.\n";
return false;
}
network::mojom::URLResponseHead const* head = ldr->ResponseInfo();
if (!head) {
LOG(INFO) << "ProcessResponse(" << gw.url() << ") Null head.\n";
return false;
}
DCHECK(gw.url().find("?format=") < gw.url().size() || gw.accept().size() > 0);
std::string reported_content_type;
head->headers->EnumerateHeader(nullptr, "Content-Type",
&reported_content_type);
// application/vnd.ipld.raw
// -- OR --
// application/vnd.ipfs.ipns-record
constexpr std::string_view content_type_prefix{"application/vnd.ip"};
if (reported_content_type.compare(0, content_type_prefix.size(),
content_type_prefix)) {
VLOG(1) << '\n'
<< gw.url() << " reported a content type of "
<< reported_content_type
<< " strongly implying that it's a full request, not a single "
"block. TODO: Remove "
<< gw->url_prefix() << " from list of gateways?\n";
state_->gateways().demote(gw->url_prefix());
return false;
}
auto cid_str = gw.task();
cid_str.erase(0, 5); // ipfs/
auto qmark = cid_str.find('?');
if (qmark < cid_str.size()) {
cid_str.erase(qmark);
}
if (state_->storage().Get(cid_str)) {
// LOG(INFO) << "Got multiple successful responses for " << cid_str;
return true;
}
auto cid = libp2p::multi::ContentIdentifierCodec::fromString(cid_str);
if (!cid.has_value()) {
LOG(ERROR) << "Invalid CID '" << cid_str << "'.";
return false;
}
auto duration = (end_time - start_time).InMillisecondsRoundedUp();
if (cid.value().content_type ==
libp2p::multi::MulticodecType::Code::LIBP2P_KEY) {
auto* bytes = reinterpret_cast<Byte const*>(body->data());
// auto record = ValidateIpnsRecord({bytes, body->size()},
// as_peer.value(), crypto_api::VerifySignature, ParseCborIpns);
auto record = ValidateIpnsRecord({bytes, body->size()}, cid.value(), *this);
if (!record.has_value()) {
LOG(ERROR) << "IPNS record failed to validate! From: " << gw.url();
return false;
}
LOG(INFO) << "IPNS record from " << gw.url() << " points " << cid_str
<< " to " << record.value().value;
ValidatedIpns entry{record.value()};
entry.resolution_ms = duration;
entry.gateway_source = gw->url_prefix();
state_->names().AssignName(cid_str, std::move(entry));
scheduler().IssueRequests(shared_from_this());
return true;
} else {
Block block{cid.value(), *body};
if (block.cid_matches_data()) {
head->headers->SetHeader("Block-Source",
cid_str + ", " + gw->url_prefix() + " @" +
std::to_string(std::time(nullptr)));
VLOG(1) << "L3: Storing CID=" << cid_str;
// Note this header is added _after_ storing in long-term cache
head->headers->SetHeader(
"Server-Timing",
"gateway-fetch-" + cid_str + ";desc=\"" + gw->url_prefix() +
" : load over http(s)\";dur=" + std::to_string(duration));
state_->storage().Store(cid_str, cid.value(),
head->headers->raw_headers(), *body);
auto& orc = state_->orchestrator();
orc.add_node(cid_str, ipld::DagNode::fromBlock(block));
if (gw.srcreq) {
if (gw.srcreq->dependent->ready_after()) {
orc.build_response(gw.srcreq->dependent);
}
} else {
LOG(ERROR) << "This BusyGateway with response has no top-level "
"IpfsRequest associated with it "
<< gw.url() << " " << gw.accept();
}
scheduler().IssueRequests(shared_from_this());
return true;
} else {
LOG(ERROR) << "You tried to store some bytes as a block for a CID ("
<< cid_str << ") that does not correspond to those bytes!";
// TODO ban the gateway outright
return false;
}
}
}

auto Self::scheduler() -> Scheduler& {
return sched_;
}
void Self::SetLoaderFactory(network::mojom::URLLoaderFactory& lf) {
loader_factory_ = &lf;
if (disc_cb_) {
Expand Down Expand Up @@ -308,15 +131,6 @@ std::string Self::UnescapeUrlComponent(std::string_view comp) const {
VLOG(1) << "UnescapeUrlComponent(" << comp << ")->'" << result << "'";
return result;
}
void Self::RequestByCid(std::string cid,
std::shared_ptr<DagListener> listener,
Priority prio) {
auto me = shared_from_this();
LOG(ERROR) << "Look out! RequestByCid(" << cid << ",...," << prio << ')';
sched_.Enqueue(me, listener, {}, "ipfs/" + cid, "application/vnd.ipld.raw",
prio, {});
sched_.IssueRequests(me);
}
void Self::SendDnsTextRequest(std::string host,
DnsTextResultsCallback res,
DnsTextCompleteCallback don) {
Expand Down Expand Up @@ -352,9 +166,7 @@ bool Self::verify_key_signature(SigningKeyType t,
Self::ChromiumIpfsContext(
InterRequestState& state,
raw_ptr<network::mojom::NetworkContext> network_context)
: network_context_{network_context},
state_{state},
sched_([this]() { return state_->gateways().GenerateList(); }) {}
: network_context_{network_context}, state_{state} {}
Self::~ChromiumIpfsContext() {
LOG(WARNING) << "API dtor - are all URIs loaded?";
}
Expand Down
9 changes: 0 additions & 9 deletions component/chromium_ipfs_context.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@

#include <ipfs_client/block_storage.h>
#include <ipfs_client/context_api.h>
#include <ipfs_client/scheduler.h>

#include <base/memory/raw_ref.h>
#include <base/time/time.h>
Expand Down Expand Up @@ -37,12 +36,9 @@ class ChromiumIpfsContext final : public ContextApi {
raw_ptr<network::mojom::URLLoaderFactory> loader_factory_ = nullptr;
raw_ptr<network::mojom::NetworkContext> network_context_;
raw_ref<InterRequestState> state_;
Scheduler sched_;
std::function<void(std::vector<std::string>)> disc_cb_;
std::map<std::string, std::unique_ptr<DnsTxtRequest>> dns_reqs_;

void Request(std::string task, std::shared_ptr<DagListener>, Priority);
std::shared_ptr<GatewayRequest> InitiateGatewayRequest(BusyGateway) override;
std::string MimeType(std::string extension,
std::string_view content,
std::string const& url) const override;
Expand All @@ -68,16 +64,11 @@ class ChromiumIpfsContext final : public ContextApi {
base::TimeTicks);
friend class NetworkRequestor;

void RequestByCid(std::string cid,
std::shared_ptr<DagListener>,
Priority);

public:
ChromiumIpfsContext(InterRequestState&,
raw_ptr<network::mojom::NetworkContext> network_context);
~ChromiumIpfsContext();
void SetLoaderFactory(network::mojom::URLLoaderFactory&);
Scheduler& scheduler();
void Discover(std::function<void(std::vector<std::string>)>) override;
};

Expand Down
Loading

0 comments on commit b40c99a

Please sign in to comment.