blob: 84ef76d2d227059ef0c5fc2387194a910abfd387 [file] [log] [blame]
// Copyright 2017 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
#include "net/reporting/reporting_cache.h"
#include <algorithm>
#include <map>
#include <set>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <utility>
#include <vector>
#include "base/stl_util.h"
#include "base/time/tick_clock.h"
#include "base/time/time.h"
#include "net/log/net_log.h"
#include "net/reporting/reporting_client.h"
#include "net/reporting/reporting_context.h"
#include "net/reporting/reporting_report.h"
#include "url/gurl.h"
namespace net {
namespace {
// Returns the superdomain of a given domain, or the empty string if the given
// domain is just a single label. Note that this does not take into account
// anything like the Public Suffix List, so the superdomain may end up being a
// bare TLD.
//
// Examples:
//
// GetSuperdomain("assets.example.com") -> "example.com"
// GetSuperdomain("example.net") -> "net"
// GetSuperdomain("littlebox") -> ""
std::string GetSuperdomain(const std::string& domain) {
size_t dot_pos = domain.find('.');
if (dot_pos == std::string::npos)
return "";
return domain.substr(dot_pos + 1);
}
struct ClientMetadata {
base::TimeTicks last_used;
ReportingCache::ClientStatistics stats;
};
class ReportingCacheImpl : public ReportingCache {
public:
ReportingCacheImpl(ReportingContext* context) : context_(context) {
DCHECK(context_);
}
~ReportingCacheImpl() override {
base::TimeTicks now = tick_clock()->NowTicks();
// Mark all undoomed reports as erased at shutdown, and record outcomes of
// all remaining reports (doomed or not).
for (auto it = reports_.begin(); it != reports_.end(); ++it) {
ReportingReport* report = it->second.get();
if (!base::ContainsKey(doomed_reports_, report))
report->outcome = ReportingReport::Outcome::ERASED_REPORTING_SHUT_DOWN;
report->RecordOutcome(now);
}
reports_.clear();
}
void AddReport(const GURL& url,
const std::string& user_agent,
const std::string& group,
const std::string& type,
std::unique_ptr<const base::Value> body,
int depth,
base::TimeTicks queued,
int attempts) override {
auto report = std::make_unique<ReportingReport>(
url, user_agent, group, type, std::move(body), depth, queued, attempts);
auto inserted =
reports_.insert(std::make_pair(report.get(), std::move(report)));
DCHECK(inserted.second);
if (reports_.size() > context_->policy().max_report_count) {
// There should be at most one extra report (the one added above).
DCHECK_EQ(context_->policy().max_report_count + 1, reports_.size());
const ReportingReport* to_evict = FindReportToEvict();
DCHECK_NE(nullptr, to_evict);
// The newly-added report isn't pending, so even if all other reports are
// pending, the cache should have a report to evict.
DCHECK(!base::ContainsKey(pending_reports_, to_evict));
reports_[to_evict]->outcome = ReportingReport::Outcome::ERASED_EVICTED;
RemoveReportInternal(to_evict);
}
context_->NotifyCacheUpdated();
}
void GetReports(
std::vector<const ReportingReport*>* reports_out) const override {
reports_out->clear();
for (const auto& it : reports_) {
if (!base::ContainsKey(doomed_reports_, it.first))
reports_out->push_back(it.second.get());
}
}
base::Value GetReportsAsValue() const override {
// Sort the queued reports by origin and timestamp.
std::vector<const ReportingReport*> sorted_reports;
sorted_reports.reserve(reports_.size());
for (const auto& it : reports_) {
sorted_reports.push_back(it.second.get());
}
std::sort(
sorted_reports.begin(), sorted_reports.end(),
[](const ReportingReport* report1, const ReportingReport* report2) {
if (report1->queued < report2->queued)
return true;
else if (report1->queued > report2->queued)
return false;
else
return report1->url < report2->url;
});
std::vector<base::Value> report_list;
for (const ReportingReport* report : sorted_reports) {
base::Value report_dict(base::Value::Type::DICTIONARY);
#if defined(STARBOARD)
report_dict.SetKey("url", base::Value(report->url.spec().c_str()));
report_dict.SetKey("group", base::Value(report->group.c_str()));
report_dict.SetKey("type", base::Value(report->type.c_str()));
#else
report_dict.SetKey("url", base::Value(report->url.spec()));
report_dict.SetKey("group", base::Value(report->group));
report_dict.SetKey("type", base::Value(report->type));
#endif
report_dict.SetKey("depth", base::Value(report->depth));
report_dict.SetKey(
"queued", base::Value(NetLog::TickCountToString(report->queued)));
report_dict.SetKey("attempts", base::Value(report->attempts));
if (report->body) {
report_dict.SetKey("body", report->body->Clone());
}
if (base::ContainsKey(doomed_reports_, report)) {
report_dict.SetKey("status", base::Value("doomed"));
} else if (base::ContainsKey(pending_reports_, report)) {
report_dict.SetKey("status", base::Value("pending"));
} else {
report_dict.SetKey("status", base::Value("queued"));
}
report_list.push_back(std::move(report_dict));
}
return base::Value(std::move(report_list));
}
void GetNonpendingReports(
std::vector<const ReportingReport*>* reports_out) const override {
reports_out->clear();
for (const auto& it : reports_) {
if (!base::ContainsKey(pending_reports_, it.first) &&
!base::ContainsKey(doomed_reports_, it.first)) {
reports_out->push_back(it.second.get());
}
}
}
void SetReportsPending(
const std::vector<const ReportingReport*>& reports) override {
for (const ReportingReport* report : reports) {
auto inserted = pending_reports_.insert(report);
DCHECK(inserted.second);
}
}
void ClearReportsPending(
const std::vector<const ReportingReport*>& reports) override {
std::vector<const ReportingReport*> reports_to_remove;
for (const ReportingReport* report : reports) {
size_t erased = pending_reports_.erase(report);
DCHECK_EQ(1u, erased);
if (base::ContainsKey(doomed_reports_, report)) {
reports_to_remove.push_back(report);
doomed_reports_.erase(report);
}
}
for (const ReportingReport* report : reports_to_remove)
RemoveReportInternal(report);
}
void IncrementReportsAttempts(
const std::vector<const ReportingReport*>& reports) override {
for (const ReportingReport* report : reports) {
DCHECK(base::ContainsKey(reports_, report));
reports_[report]->attempts++;
}
context_->NotifyCacheUpdated();
}
void IncrementEndpointDeliveries(const url::Origin& origin,
const GURL& endpoint,
int reports_delivered,
bool successful) override {
const ReportingClient* client =
GetClientByOriginAndEndpoint(origin, endpoint);
if (client) {
auto& metadata = client_metadata_[client];
metadata.stats.attempted_uploads++;
metadata.stats.attempted_reports += reports_delivered;
if (successful) {
metadata.stats.successful_uploads++;
metadata.stats.successful_reports += reports_delivered;
}
}
}
void RemoveReports(const std::vector<const ReportingReport*>& reports,
ReportingReport::Outcome outcome) override {
for (const ReportingReport* report : reports) {
reports_[report]->outcome = outcome;
if (base::ContainsKey(pending_reports_, report)) {
doomed_reports_.insert(report);
} else {
DCHECK(!base::ContainsKey(doomed_reports_, report));
RemoveReportInternal(report);
}
}
context_->NotifyCacheUpdated();
}
void RemoveAllReports(ReportingReport::Outcome outcome) override {
std::vector<const ReportingReport*> reports_to_remove;
for (auto it = reports_.begin(); it != reports_.end(); ++it) {
ReportingReport* report = it->second.get();
report->outcome = outcome;
if (!base::ContainsKey(pending_reports_, report))
reports_to_remove.push_back(report);
else
doomed_reports_.insert(report);
}
for (const ReportingReport* report : reports_to_remove)
RemoveReportInternal(report);
context_->NotifyCacheUpdated();
}
void SetClient(const url::Origin& origin,
const GURL& endpoint,
ReportingClient::Subdomains subdomains,
const std::string& group,
base::TimeTicks expires,
int priority,
int weight) override {
DCHECK(endpoint.SchemeIsCryptographic());
base::TimeTicks last_used = tick_clock()->NowTicks();
const ReportingClient* old_client =
GetClientByOriginAndEndpoint(origin, endpoint);
if (old_client) {
last_used = client_metadata_[old_client].last_used;
RemoveClient(old_client);
}
AddClient(
std::make_unique<ReportingClient>(origin, endpoint, subdomains, group,
expires, priority, weight),
last_used);
if (client_metadata_.size() > context_->policy().max_client_count) {
// There should only ever be one extra client, added above.
DCHECK_EQ(context_->policy().max_client_count + 1,
client_metadata_.size());
// And that shouldn't happen if it was replaced, not added.
DCHECK(!old_client);
const ReportingClient* to_evict =
FindClientToEvict(tick_clock()->NowTicks());
DCHECK(to_evict);
RemoveClient(to_evict);
}
context_->NotifyCacheUpdated();
}
void MarkClientUsed(const ReportingClient* client) override {
DCHECK(client);
client_metadata_[client].last_used = tick_clock()->NowTicks();
}
void GetClients(
std::vector<const ReportingClient*>* clients_out) const override {
clients_out->clear();
for (const auto& it : clients_)
for (const auto& endpoint_and_client : it.second)
clients_out->push_back(endpoint_and_client.second.get());
}
base::Value GetClientsAsValue() const override {
std::map<const url::Origin,
std::map<const std::string, std::vector<const ReportingClient*>>>
clients_by_origin_and_group;
for (const auto& it : clients_) {
const url::Origin& origin = it.first;
for (const auto& endpoint_and_client : it.second) {
const ReportingClient* client = endpoint_and_client.second.get();
clients_by_origin_and_group[origin][client->group].push_back(client);
}
}
std::vector<base::Value> origin_list;
for (const auto& it : clients_by_origin_and_group) {
const url::Origin& origin = it.first;
base::Value origin_dict(base::Value::Type::DICTIONARY);
origin_dict.SetKey("origin", base::Value(origin.Serialize()));
std::vector<base::Value> group_list;
for (const auto& group_and_clients : it.second) {
const std::string& group = group_and_clients.first;
const std::vector<const ReportingClient*>& clients =
group_and_clients.second;
base::Value group_dict(base::Value::Type::DICTIONARY);
#if defined(STARBOARD)
group_dict.SetKey("name", base::Value(group.c_str()));
#else
group_dict.SetKey("name", base::Value(group));
#endif
std::vector<base::Value> endpoint_list;
for (const ReportingClient* client : clients) {
base::Value endpoint_dict(base::Value::Type::DICTIONARY);
// Reporting defines the group as a whole to have an expiration time
// and subdomains flag, not the individual endpoints within the group.
group_dict.SetKey(
"expires",
#if defined(STARBOARD)
base::Value(NetLog::TickCountToString(client->expires).c_str()));
#else
base::Value(NetLog::TickCountToString(client->expires)));
#endif
group_dict.SetKey("includeSubdomains",
base::Value(client->subdomains ==
ReportingClient::Subdomains::INCLUDE));
endpoint_dict.SetKey("url", base::Value(client->endpoint.spec()));
endpoint_dict.SetKey("priority", base::Value(client->priority));
endpoint_dict.SetKey("weight", base::Value(client->weight));
auto metadata_it = client_metadata_.find(client);
if (metadata_it != client_metadata_.end()) {
const ClientStatistics& stats = metadata_it->second.stats;
base::Value successful_dict(base::Value::Type::DICTIONARY);
successful_dict.SetKey("uploads",
base::Value(stats.successful_uploads));
successful_dict.SetKey("reports",
base::Value(stats.successful_reports));
endpoint_dict.SetKey("successful", std::move(successful_dict));
base::Value failed_dict(base::Value::Type::DICTIONARY);
failed_dict.SetKey("uploads",
base::Value(stats.attempted_uploads -
stats.successful_uploads));
failed_dict.SetKey("reports",
base::Value(stats.attempted_reports -
stats.successful_reports));
endpoint_dict.SetKey("failed", std::move(failed_dict));
}
endpoint_list.push_back(std::move(endpoint_dict));
}
group_dict.SetKey("endpoints", base::Value(std::move(endpoint_list)));
group_list.push_back(std::move(group_dict));
}
origin_dict.SetKey("groups", base::Value(std::move(group_list)));
origin_list.push_back(std::move(origin_dict));
}
return base::Value(std::move(origin_list));
}
void GetClientsForOriginAndGroup(
const url::Origin& origin,
const std::string& group,
std::vector<const ReportingClient*>* clients_out) const override {
clients_out->clear();
const auto it = clients_.find(origin);
if (it != clients_.end()) {
for (const auto& endpoint_and_client : it->second) {
if (endpoint_and_client.second->group == group)
clients_out->push_back(endpoint_and_client.second.get());
}
}
// If no clients were found, try successive superdomain suffixes until a
// client with include_subdomains is found or there are no more domain
// components left.
std::string domain = origin.host();
while (clients_out->empty() && !domain.empty()) {
GetWildcardClientsForDomainAndGroup(domain, group, clients_out);
domain = GetSuperdomain(domain);
}
}
// TODO(juliatuttle): Unittests.
void GetEndpointsForOrigin(const url::Origin& origin,
std::vector<GURL>* endpoints_out) const override {
endpoints_out->clear();
const auto it = clients_.find(origin);
if (it == clients_.end())
return;
for (const auto& endpoint_and_client : it->second)
endpoints_out->push_back(endpoint_and_client.first);
}
void RemoveClients(
const std::vector<const ReportingClient*>& clients_to_remove) override {
for (const ReportingClient* client : clients_to_remove)
RemoveClient(client);
context_->NotifyCacheUpdated();
}
void RemoveClientForOriginAndEndpoint(const url::Origin& origin,
const GURL& endpoint) override {
const ReportingClient* client =
GetClientByOriginAndEndpoint(origin, endpoint);
RemoveClient(client);
context_->NotifyCacheUpdated();
}
void RemoveClientsForEndpoint(const GURL& endpoint) override {
std::vector<const ReportingClient*> clients_to_remove;
for (auto& origin_and_endpoints : clients_)
if (base::ContainsKey(origin_and_endpoints.second, endpoint))
clients_to_remove.push_back(
origin_and_endpoints.second[endpoint].get());
for (const ReportingClient* client : clients_to_remove)
RemoveClient(client);
if (!clients_to_remove.empty())
context_->NotifyCacheUpdated();
}
void RemoveAllClients() override {
clients_.clear();
wildcard_clients_.clear();
client_metadata_.clear();
context_->NotifyCacheUpdated();
}
ClientStatistics GetStatisticsForOriginAndEndpoint(
const url::Origin& origin,
const GURL& endpoint) const override {
const ReportingClient* client =
GetClientByOriginAndEndpoint(origin, endpoint);
auto it = client_metadata_.find(client);
if (it == client_metadata_.end()) {
return ClientStatistics();
}
return it->second.stats;
}
size_t GetFullReportCountForTesting() const override {
return reports_.size();
}
bool IsReportPendingForTesting(const ReportingReport* report) const override {
return base::ContainsKey(pending_reports_, report);
}
bool IsReportDoomedForTesting(const ReportingReport* report) const override {
return base::ContainsKey(doomed_reports_, report);
}
private:
void RemoveReportInternal(const ReportingReport* report) {
reports_[report]->RecordOutcome(tick_clock()->NowTicks());
size_t erased = reports_.erase(report);
DCHECK_EQ(1u, erased);
}
const ReportingReport* FindReportToEvict() const {
const ReportingReport* earliest_queued = nullptr;
for (const auto& it : reports_) {
const ReportingReport* report = it.first;
if (base::ContainsKey(pending_reports_, report))
continue;
if (!earliest_queued || report->queued < earliest_queued->queued) {
earliest_queued = report;
}
}
return earliest_queued;
}
void AddClient(std::unique_ptr<ReportingClient> client,
base::TimeTicks last_used) {
DCHECK(client);
url::Origin origin = client->origin;
GURL endpoint = client->endpoint;
auto inserted_metadata = client_metadata_.insert(
std::make_pair(client.get(), ClientMetadata{last_used}));
DCHECK(inserted_metadata.second);
if (client->subdomains == ReportingClient::Subdomains::INCLUDE) {
const std::string& domain = origin.host();
auto inserted_wildcard_client =
wildcard_clients_[domain].insert(client.get());
DCHECK(inserted_wildcard_client.second);
}
auto inserted_client =
clients_[origin].insert(std::make_pair(endpoint, std::move(client)));
DCHECK(inserted_client.second);
}
void RemoveClient(const ReportingClient* client) {
DCHECK(client);
url::Origin origin = client->origin;
GURL endpoint = client->endpoint;
if (client->subdomains == ReportingClient::Subdomains::INCLUDE) {
const std::string& domain = origin.host();
size_t erased_wildcard_client = wildcard_clients_[domain].erase(client);
DCHECK_EQ(1u, erased_wildcard_client);
if (wildcard_clients_[domain].empty()) {
size_t erased_wildcard_domain = wildcard_clients_.erase(domain);
DCHECK_EQ(1u, erased_wildcard_domain);
}
}
size_t erased_metadata = client_metadata_.erase(client);
DCHECK_EQ(1u, erased_metadata);
size_t erased_endpoint = clients_[origin].erase(endpoint);
DCHECK_EQ(1u, erased_endpoint);
if (clients_[origin].empty()) {
size_t erased_origin = clients_.erase(origin);
DCHECK_EQ(1u, erased_origin);
}
}
const ReportingClient* GetClientByOriginAndEndpoint(
const url::Origin& origin,
const GURL& endpoint) const {
const auto& origin_it = clients_.find(origin);
if (origin_it == clients_.end())
return nullptr;
const auto& endpoint_it = origin_it->second.find(endpoint);
if (endpoint_it == origin_it->second.end())
return nullptr;
return endpoint_it->second.get();
}
void GetWildcardClientsForDomainAndGroup(
const std::string& domain,
const std::string& group,
std::vector<const ReportingClient*>* clients_out) const {
clients_out->clear();
auto it = wildcard_clients_.find(domain);
if (it == wildcard_clients_.end())
return;
for (const ReportingClient* client : it->second) {
DCHECK_EQ(ReportingClient::Subdomains::INCLUDE, client->subdomains);
if (client->group == group)
clients_out->push_back(client);
}
}
const ReportingClient* FindClientToEvict(base::TimeTicks now) const {
DCHECK(!client_metadata_.empty());
const ReportingClient* earliest_used = nullptr;
base::TimeTicks earliest_used_last_used;
const ReportingClient* earliest_expired = nullptr;
for (const auto& it : client_metadata_) {
const ReportingClient* client = it.first;
base::TimeTicks client_last_used = it.second.last_used;
if (earliest_used == nullptr ||
client_last_used < earliest_used_last_used) {
earliest_used = client;
earliest_used_last_used = client_last_used;
}
if (earliest_expired == nullptr ||
client->expires < earliest_expired->expires) {
earliest_expired = client;
}
}
// If there are expired clients, return the earliest-expired.
if (earliest_expired->expires < now)
return earliest_expired;
else
return earliest_used;
}
const base::TickClock* tick_clock() { return context_->tick_clock(); }
ReportingContext* context_;
// Owns all reports, keyed by const raw pointer for easier lookup.
std::unordered_map<const ReportingReport*, std::unique_ptr<ReportingReport>>
reports_;
// Reports that have been marked pending (in use elsewhere and should not be
// deleted until no longer pending).
std::unordered_set<const ReportingReport*> pending_reports_;
// Reports that have been marked doomed (would have been deleted, but were
// pending when the deletion was requested).
std::unordered_set<const ReportingReport*> doomed_reports_;
// Owns all clients, keyed by origin, then endpoint URL.
// (These would be unordered_map, but neither url::Origin nor GURL has a hash
// function implemented.)
std::map<url::Origin, std::map<GURL, std::unique_ptr<ReportingClient>>>
clients_;
// References but does not own all clients with include_subdomains set, keyed
// by domain name.
std::unordered_map<std::string, std::unordered_set<const ReportingClient*>>
wildcard_clients_;
// The time that each client has last been used.
std::unordered_map<const ReportingClient*, ClientMetadata> client_metadata_;
};
} // namespace
// static
std::unique_ptr<ReportingCache> ReportingCache::Create(
ReportingContext* context) {
return std::make_unique<ReportingCacheImpl>(context);
}
ReportingCache::~ReportingCache() = default;
} // namespace net