blob: d0777de672945d8375eb91a393c9e77225873bfb [file] [log] [blame]
// Copyright 2015 The Cobalt Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include <memory>
#include <ostream>
#include <sstream>
#include <vector>
#include "base/command_line.h"
#include "base/files/file_util.h"
#include "base/json/json_reader.h"
#include "base/message_loop/message_loop.h"
#include "base/run_loop.h"
#include "base/strings/string_util.h"
#include "base/test/scoped_task_environment.h"
#include "base/values.h"
#include "cobalt/browser/web_module.h"
#include "cobalt/cssom/viewport_size.h"
#include "cobalt/dom/csp_delegate_factory.h"
#include "cobalt/layout_tests/test_utils.h"
#include "cobalt/layout_tests/web_platform_test_parser.h"
#include "cobalt/math/size.h"
#include "cobalt/media/media_module.h"
#include "cobalt/network/network_module.h"
#include "cobalt/render_tree/resource_provider_stub.h"
#include "starboard/window.h"
#include "testing/gtest/include/gtest/gtest.h"
#include "url/gurl.h"
using cobalt::cssom::ViewportSize;
namespace cobalt {
namespace layout_tests {
namespace {
// A CspDelegate that behaves more like a "standard" one. i.e. it
// is permissive by default. This is for testing our compliance
// with the web-platform-test suite.
class CspDelegatePermissive : public dom::CspDelegateSecure {
public:
CspDelegatePermissive(
std::unique_ptr<dom::CspViolationReporter> violation_reporter,
const GURL& url, csp::CSPHeaderPolicy require_csp,
const base::Closure& policy_changed_callback)
: dom::CspDelegateSecure(std::move(violation_reporter), url, require_csp,
policy_changed_callback) {
// Lies, but some checks in our parent require this.
was_header_received_ = true;
}
static CspDelegate* Create(
std::unique_ptr<dom::CspViolationReporter> violation_reporter,
const GURL& url, csp::CSPHeaderPolicy require_csp,
const base::Closure& policy_changed_callback,
int insecure_allowed_token) {
return new CspDelegatePermissive(std::move(violation_reporter), url,
require_csp, policy_changed_callback);
}
bool OnReceiveHeaders(const csp::ResponseHeaders& headers) override {
csp_->OnReceiveHeaders(headers);
if (!policy_changed_callback_.is_null()) {
policy_changed_callback_.Run();
}
// |true| to allow the document to continue loading.
return true;
}
};
// Match the enums from testharness.js.
enum TestStatus {
kPass = 0,
kFail,
kTimeout,
kNotrun,
};
enum TestsStatus {
kTestsOk = 0,
kTestsError,
kTestsTimeout
};
std::string TestStatusToString(int status) {
switch (status) {
case kPass:
return "PASS";
case kFail:
return "FAIL";
case kTimeout:
return "TIMEOUT";
case kNotrun:
return "NOTRUN";
}
NOTREACHED();
return "FAIL";
}
std::string TestsStatusToString(int status) {
switch (status) {
case kTestsOk:
return "OK";
case kTestsError:
return "ERROR";
case kTestsTimeout:
return "TIMEOUT";
}
NOTREACHED();
return "FAIL";
}
struct TestResult {
std::string name;
int status;
std::string message;
std::string stack;
};
struct HarnessResult {
std::vector<TestResult> test_results;
int status;
std::string message;
};
const char* kLogSuppressions[] = {
"<link> has unsupported rel value: help.",
"<link> has unsupported rel value: author.",
"synchronous XHR is not supported",
};
void Quit(base::RunLoop* run_loop) {
base::MessageLoop::current()->task_runner()->PostTask(
FROM_HERE, run_loop->QuitClosure());
}
// Called when layout completes and results have been produced. We use this
// signal to stop the WebModule's message loop since our work is done after a
// layout has been performed.
void WebModuleOnRenderTreeProducedCallback(
base::Optional<browser::WebModule::LayoutResults>* out_results,
base::RunLoop* run_loop, base::MessageLoop* message_loop,
const browser::WebModule::LayoutResults& results) {
out_results->emplace(results.render_tree, results.layout_time);
message_loop->task_runner()->PostTask(FROM_HERE, base::Bind(Quit, run_loop));
}
// This callback, when called, quits a message loop, outputs the error message
// and sets the success flag to false.
void WebModuleErrorCallback(base::RunLoop* run_loop,
base::MessageLoop* message_loop, const GURL& url,
const std::string& error) {
LOG(FATAL) << "Error loading document: " << error << ". URL: " << url;
message_loop->task_runner()->PostTask(FROM_HERE, base::Bind(Quit, run_loop));
}
std::string RunWebPlatformTest(const GURL& url, bool* got_results) {
LogFilter log_filter;
for (size_t i = 0; i < arraysize(kLogSuppressions); ++i) {
log_filter.Add(kLogSuppressions[i]);
}
// Setup a message loop for the current thread since we will be constructing
// a WebModule, which requires a message loop to exist for the current
// thread.
base::test::ScopedTaskEnvironment scoped_task_environment;
const ViewportSize kDefaultViewportSize(640, 360);
// Setup WebModule's auxiliary components.
// Network module
network::NetworkModule::Options net_options;
net_options.https_requirement = network::kHTTPSOptional;
std::string custom_proxy =
base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII("proxy");
if (!custom_proxy.empty()) net_options.custom_proxy = custom_proxy;
network::NetworkModule network_module(net_options);
// Media module
render_tree::ResourceProviderStub resource_provider;
std::unique_ptr<media::MediaModule> media_module(
new media::MediaModule(NULL, &resource_provider));
std::unique_ptr<media::CanPlayTypeHandler> can_play_type_handler(
media::MediaModule::CreateCanPlayTypeHandler());
dom::CspDelegateFactory::GetInstance()->OverrideCreator(
dom::kCspEnforcementEnable, CspDelegatePermissive::Create);
// Use test runner mode to allow the content itself to dictate when it is
// ready for layout should be performed. See cobalt/dom/test_runner.h.
browser::WebModule::Options web_module_options;
web_module_options.layout_trigger = layout::LayoutManager::kTestRunnerMode;
// We assume that we won't suspend/resume while running the tests, and so
// we take advantage of the convenience of inline script tags.
web_module_options.enable_inline_script_warnings = false;
// Prepare a slot for our results to be placed when ready.
base::Optional<browser::WebModule::LayoutResults> results;
base::RunLoop run_loop;
// Create the WebModule and wait for a layout to occur.
browser::WebModule web_module(
url, base::kApplicationStateStarted,
base::Bind(&WebModuleOnRenderTreeProducedCallback, &results, &run_loop,
base::MessageLoop::current()),
base::Bind(&WebModuleErrorCallback, &run_loop,
base::MessageLoop::current()),
browser::WebModule::CloseCallback() /* window_close_callback */,
base::Closure() /* window_minimize_callback */,
can_play_type_handler.get(), media_module.get(), &network_module,
kDefaultViewportSize, 1.f, &resource_provider, 60.0f, web_module_options);
run_loop.Run();
const std::string extract_results =
"document.getElementById(\"__testharness__results__\").textContent;";
std::string output = web_module.ExecuteJavascript(
extract_results, base::SourceLocation(__FILE__, __LINE__, 1),
got_results);
return output;
}
HarnessResult ParseResults(const std::string& json_results) {
HarnessResult harness_result;
std::vector<TestResult>& test_results = harness_result.test_results;
std::unique_ptr<base::Value> root;
base::JSONReader reader(
base::JSONParserOptions::JSON_REPLACE_INVALID_CHARACTERS);
root = reader.ReadToValue(json_results);
// Expect that parsing test result succeeded.
EXPECT_EQ(base::JSONReader::JSON_NO_ERROR, reader.error_code());
if (!root) {
// Unparseable JSON, or empty string.
LOG(ERROR) << "Web Platform Tests returned unparseable JSON test result!";
return harness_result;
}
base::DictionaryValue* root_as_dict;
EXPECT_EQ(true, root->GetAsDictionary(&root_as_dict));
EXPECT_EQ(true, root_as_dict->GetInteger("status", &harness_result.status));
// "message" field might not be set
root_as_dict->GetString("message", &harness_result.message);
base::ListValue* test_list;
EXPECT_EQ(true, root_as_dict->GetList("tests", &test_list));
for (size_t i = 0; i < test_list->GetSize(); ++i) {
TestResult result;
base::DictionaryValue* test_dict;
EXPECT_EQ(true, test_list->GetDictionary(i, &test_dict));
EXPECT_EQ(true, test_dict->GetInteger("status", &result.status));
EXPECT_EQ(true, test_dict->GetString("name", &result.name));
// These fields may be null.
test_dict->GetString("message", &result.message);
test_dict->GetString("stack", &result.stack);
test_results.push_back(result);
}
return harness_result;
}
::testing::AssertionResult CheckResult(const char* expectation_str,
const char* results_str,
bool should_pass,
const TestResult& result) {
bool test_passed = result.status == WebPlatformTestInfo::kPass;
if (test_passed == should_pass) {
return ::testing::AssertionSuccess();
} else {
std::ostringstream output;
output << std::endl
<< "Test \"" << result.name
<< "\" status: " << TestStatusToString(result.status);
if (!result.message.empty()) {
output << std::endl << result.message;
}
if (!result.stack.empty()) {
output << std::endl << result.stack;
}
WebPlatformTestInfo::State expectation =
should_pass ? WebPlatformTestInfo::kPass : WebPlatformTestInfo::kFail;
return ::testing::AssertionFailure()
<< "Expected: " << TestStatusToString(expectation) << std::endl
<< "Actual: " << TestStatusToString(result.status) << std::endl
<< output.str();
}
}
::testing::AssertionResult CheckHarnessResult(
const char* expectation_str, const char* results_str,
WebPlatformTestInfo::State expect_status, const HarnessResult& result) {
if ((expect_status == WebPlatformTestInfo::State::kPass) &&
(result.status != kTestsOk)) {
return ::testing::AssertionFailure()
<< " Harness status :" << TestsStatusToString(result.status)
<< std::endl
<< result.message;
} else {
return ::testing::AssertionSuccess();
}
}
struct GetTestName {
std::string operator()(
const ::testing::TestParamInfo<WebPlatformTestInfo>& info) const {
// Only alphanumeric characters and '_' are valid.
std::string name = info.param.url;
for (size_t i = 0; i < name.size(); ++i) {
char ch = name[i];
if (ch >= 'A' && ch <= 'Z') {
continue;
}
if (ch >= 'a' && ch <= 'z') {
continue;
}
if (ch >= '0' && ch <= '9') {
continue;
}
name[i] = '_';
}
return name;
}
};
} // namespace
class WebPlatformTest : public ::testing::TestWithParam<WebPlatformTestInfo> {};
TEST_P(WebPlatformTest, Run) {
// Output the name of the current input file so that it is visible in test
// output.
std::string test_server =
base::CommandLine::ForCurrentProcess()->GetSwitchValueASCII(
"web-platform-test-server");
if (test_server.empty()) test_server = "http://web-platform.test:8000";
GURL test_url = GURL(test_server).Resolve(GetParam().url);
LOG(INFO) << "(" << test_url << ")" << std::endl;
bool got_results = false;
std::string json_results = RunWebPlatformTest(test_url, &got_results);
EXPECT_TRUE(got_results);
HarnessResult result = ParseResults(json_results);
const std::vector<TestResult>& results = result.test_results;
bool failed_at_least_once = false;
for (size_t i = 0; i < results.size(); ++i) {
const WebPlatformTestInfo& test_info = GetParam();
const TestResult& test_result = results[i];
bool should_pass = test_info.expectation == WebPlatformTestInfo::kPass;
std::set<std::string>::const_iterator it =
test_info.exceptions.find(test_result.name);
if (it != test_info.exceptions.end()) {
should_pass = !should_pass;
}
// If expected to fail but current subtest did not fail, wait to report
// the entire test failed after the last subtest passed and none failed.
if (!should_pass && test_result.status == WebPlatformTestInfo::kPass &&
(i != results.size() - 1 || failed_at_least_once)) {
should_pass = true;
} else {
failed_at_least_once = failed_at_least_once ||
test_result.status == WebPlatformTestInfo::kFail;
}
EXPECT_PRED_FORMAT2(CheckResult, should_pass, test_result);
}
EXPECT_PRED_FORMAT2(CheckHarnessResult, GetParam().expectation, result);
}
// Disable on Windows until network stack is implemented.
#if !defined(COBALT_WIN)
// XML Http Request test cases.
INSTANTIATE_TEST_CASE_P(
xhr, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("XMLHttpRequest")),
GetTestName());
INSTANTIATE_TEST_CASE_P(
cobalt_special, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("cobalt_special")),
GetTestName());
INSTANTIATE_TEST_CASE_P(
csp, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("content-security-policy")),
GetTestName());
INSTANTIATE_TEST_CASE_P(dom, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("dom")),
GetTestName());
INSTANTIATE_TEST_CASE_P(cors, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("cors")),
GetTestName());
INSTANTIATE_TEST_CASE_P(
fetch, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("fetch", "'fetch' in this")),
GetTestName());
INSTANTIATE_TEST_CASE_P(html, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("html")),
GetTestName());
INSTANTIATE_TEST_CASE_P(
mediasession, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("mediasession")),
GetTestName());
INSTANTIATE_TEST_CASE_P(streams, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests(
"streams", "'ReadableStream' in this")),
GetTestName());
INSTANTIATE_TEST_CASE_P(webidl, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("WebIDL")),
GetTestName());
INSTANTIATE_TEST_CASE_P(websockets, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("websockets")),
GetTestName());
INSTANTIATE_TEST_CASE_P(
web_crypto_api, WebPlatformTest,
::testing::ValuesIn(EnumerateWebPlatformTests("WebCryptoAPI")),
GetTestName());
#endif // !defined(COBALT_WIN)
} // namespace layout_tests
} // namespace cobalt