| // Copyright 2016 the V8 project authors. All rights reserved. |
| // Use of this source code is governed by a BSD-style license that can be |
| // found in the LICENSE file. |
| |
| #include <memory> |
| #include <string> |
| #include <vector> |
| |
| #include "src/unicode-decoder.h" |
| #include "src/unicode-inl.h" |
| #include "testing/gtest/include/gtest/gtest.h" |
| |
| namespace v8 { |
| namespace internal { |
| |
| namespace { |
| |
| using Utf8Decoder = unibrow::Utf8Decoder<512>; |
| |
| void Decode(Utf8Decoder* decoder, const std::string& str) { |
| // Put the string in its own buffer on the heap to make sure that |
| // AddressSanitizer's heap-buffer-overflow logic can see what's going on. |
| std::unique_ptr<char[]> buffer(new char[str.length()]); |
| memcpy(buffer.get(), str.data(), str.length()); |
| decoder->Reset(buffer.get(), str.length()); |
| } |
| |
| void DecodeNormally(const std::vector<byte>& bytes, |
| std::vector<unibrow::uchar>* output) { |
| size_t cursor = 0; |
| while (cursor < bytes.size()) { |
| output->push_back( |
| unibrow::Utf8::ValueOf(bytes.data() + cursor, bytes.size(), &cursor)); |
| } |
| } |
| |
| void DecodeIncrementally(const std::vector<byte>& bytes, |
| std::vector<unibrow::uchar>* output) { |
| unibrow::Utf8::Utf8IncrementalBuffer buffer = 0; |
| for (auto b : bytes) { |
| unibrow::uchar result = unibrow::Utf8::ValueOfIncremental(b, &buffer); |
| if (result != unibrow::Utf8::kIncomplete) { |
| output->push_back(result); |
| } |
| } |
| unibrow::uchar result = unibrow::Utf8::ValueOfIncrementalFinish(&buffer); |
| if (result != unibrow::Utf8::kBufferEmpty) { |
| output->push_back(result); |
| } |
| } |
| |
| } // namespace |
| |
| TEST(UnicodeTest, ReadOffEndOfUtf8String) { |
| Utf8Decoder decoder; |
| |
| // Not enough continuation bytes before string ends. |
| Decode(&decoder, "\xE0"); |
| Decode(&decoder, "\xED"); |
| Decode(&decoder, "\xF0"); |
| Decode(&decoder, "\xF4"); |
| } |
| |
| TEST(UnicodeTest, IncrementalUTF8DecodingVsNonIncrementalUtf8Decoding) { |
| // Unfortunately, V8 has two UTF-8 decoders. This test checks that they |
| // produce the same result. This test was inspired by |
| // https://www.cl.cam.ac.uk/~mgk25/ucs/examples/UTF-8-test.txt . |
| typedef struct { |
| std::vector<byte> bytes; |
| std::vector<unibrow::uchar> unicode_expected; |
| } TestCase; |
| |
| TestCase data[] = { |
| // Correct UTF-8 text. |
| {{0xce, 0xba, 0xe1, 0xbd, 0xb9, 0xcf, 0x83, 0xce, 0xbc, 0xce, 0xb5}, |
| {0x3ba, 0x1f79, 0x3c3, 0x3bc, 0x3b5}}, |
| |
| // First possible sequence of a certain length: |
| // 1 byte |
| {{0x00}, {0x0}}, |
| // 2 bytes |
| {{0xc2, 0x80}, {0x80}}, |
| // 3 bytes |
| {{0xe0, 0xa0, 0x80}, {0x800}}, |
| // 4 bytes |
| {{0xf0, 0x90, 0x80, 0x80}, {0x10000}}, |
| // 5 bytes (not supported) |
| {{0xf8, 0x88, 0x80, 0x80, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 6 bytes (not supported) |
| {{0xfc, 0x84, 0x80, 0x80, 0x80, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Last possible sequence of certain length: |
| // 1 byte |
| {{0x7f}, {0x7f}}, |
| // 2 bytes |
| {{0xdf, 0xbf}, {0x7ff}}, |
| // 3 bytes |
| {{0xef, 0xbf, 0xbf}, {0xffff}}, |
| // 4 bytes (this sequence is not a valid code point) |
| {{0xf7, 0xbf, 0xbf, 0xbf}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 5 bytes (not supported) |
| {{0xfb, 0xbf, 0xbf, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 6 bytes (not supported) |
| {{0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // Other boundary conditions: |
| {{0xed, 0x9f, 0xbf}, {0xd7ff}}, |
| {{0xee, 0x80, 0x80}, {0xe000}}, |
| // U+fffd (invalid code point) |
| {{0xef, 0xbf, 0xbd}, {0xfffd}}, |
| // U+10ffff (last valid code point) |
| {{0xf4, 0x8f, 0xbf, 0xbf}, {0x10ffff}}, |
| // First invalid (too large) code point |
| {{0xf4, 0x90, 0x80, 0x80}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Malformed sequences: |
| // Unexpected continuation bytes: |
| // First continuation byte |
| {{0x80}, {0xfffd}}, |
| // Last continuation byte |
| {{0xbf}, {0xfffd}}, |
| // 2 continuation bytes |
| {{0x80, 0xbf}, {0xfffd, 0xfffd}}, |
| // 3 continuation bytes |
| {{0x80, 0xbf, 0x80}, {0xfffd, 0xfffd, 0xfffd}}, |
| // 4 continuation bytes |
| {{0x80, 0xbf, 0x80, 0xbf}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 5 continuation bytes |
| {{0x80, 0xbf, 0x80, 0xbf, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 6 continuation bytes |
| {{0x80, 0xbf, 0x80, 0xbf, 0x80, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 7 continuation bytes |
| {{0x80, 0xbf, 0x80, 0xbf, 0x80, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // Sequence of all 64 possible continuation bytes |
| {{0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8a, |
| 0x8b, 0x8c, 0x8d, 0x8e, 0x8f, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, |
| 0x96, 0x97, 0x98, 0x99, 0x9a, 0x9b, 0x9c, 0x9d, 0x9e, 0x9f, 0xa0, |
| 0xa1, 0xa2, 0xa3, 0xa4, 0xa5, 0xa6, 0xa7, 0xa8, 0xa9, 0xaa, 0xab, |
| 0xac, 0xad, 0xae, 0xaf, 0xb0, 0xb1, 0xb2, 0xb3, 0xb4, 0xb5, 0xb6, |
| 0xb7, 0xb8, 0xb9, 0xba, 0xbb, 0xbc, 0xbd, 0xbe, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // Using each possible continuation byte in a two-byte sequence: |
| {{0xd0, 0x80, 0xd0, 0x81, 0xd0, 0x82, 0xd0, 0x83, 0xd0, 0x84, 0xd0, 0x85, |
| 0xd0, 0x86, 0xd0, 0x87, 0xd0, 0x88, 0xd0, 0x89, 0xd0, 0x8a, 0xd0, 0x8b, |
| 0xd0, 0x8c, 0xd0, 0x8d, 0xd0, 0x8e, 0xd0, 0x8f, 0xd0, 0x90, 0xd0, 0x91, |
| 0xd0, 0x92, 0xd0, 0x93, 0xd0, 0x94, 0xd0, 0x95, 0xd0, 0x96, 0xd0, 0x97, |
| 0xd0, 0x98, 0xd0, 0x99, 0xd0, 0x9a, 0xd0, 0x9b, 0xd0, 0x9c, 0xd0, 0x9d, |
| 0xd0, 0x9e, 0xd0, 0x9f, 0xd0, 0xa0, 0xd0, 0xa1, 0xd0, 0xa2, 0xd0, 0xa3, |
| 0xd0, 0xa4, 0xd0, 0xa5, 0xd0, 0xa6, 0xd0, 0xa7, 0xd0, 0xa8, 0xd0, 0xa9, |
| 0xd0, 0xaa, 0xd0, 0xab, 0xd0, 0xac, 0xd0, 0xad, 0xd0, 0xae, 0xd0, 0xaf, |
| 0xd0, 0xb0, 0xd0, 0xb1, 0xd0, 0xb2, 0xd0, 0xb3, 0xd0, 0xb4, 0xd0, 0xb5, |
| 0xd0, 0xb6, 0xd0, 0xb7, 0xd0, 0xb8, 0xd0, 0xb9, 0xd0, 0xba, 0xd0, 0xbb, |
| 0xd0, 0xbc, 0xd0, 0xbd, 0xd0, 0xbe, 0xd0, 0xbf}, |
| {0x400, 0x401, 0x402, 0x403, 0x404, 0x405, 0x406, 0x407, 0x408, 0x409, |
| 0x40a, 0x40b, 0x40c, 0x40d, 0x40e, 0x40f, 0x410, 0x411, 0x412, 0x413, |
| 0x414, 0x415, 0x416, 0x417, 0x418, 0x419, 0x41a, 0x41b, 0x41c, 0x41d, |
| 0x41e, 0x41f, 0x420, 0x421, 0x422, 0x423, 0x424, 0x425, 0x426, 0x427, |
| 0x428, 0x429, 0x42a, 0x42b, 0x42c, 0x42d, 0x42e, 0x42f, 0x430, 0x431, |
| 0x432, 0x433, 0x434, 0x435, 0x436, 0x437, 0x438, 0x439, 0x43a, 0x43b, |
| 0x43c, 0x43d, 0x43e, 0x43f}}, |
| |
| // Lonely first bytes: |
| // All 32 first bytes of 32-byte sequences, each followed by a space |
| // (generates 32 invalid char + space sequences. |
| {{0xc0, 0x20, 0xc1, 0x20, 0xc2, 0x20, 0xc3, 0x20, 0xc4, 0x20, 0xc5, |
| 0x20, 0xc6, 0x20, 0xc7, 0x20, 0xc8, 0x20, 0xc9, 0x20, 0xca, 0x20, |
| 0xcb, 0x20, 0xcc, 0x20, 0xcd, 0x20, 0xce, 0x20, 0xcf, 0x20, 0xd0, |
| 0x20, 0xd1, 0x20, 0xd2, 0x20, 0xd3, 0x20, 0xd4, 0x20, 0xd5, 0x20, |
| 0xd6, 0x20, 0xd7, 0x20, 0xd8, 0x20, 0xd9, 0x20, 0xda, 0x20, 0xdb, |
| 0x20, 0xdc, 0x20, 0xdd, 0x20, 0xde, 0x20, 0xdf, 0x20}, |
| {0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20}}, |
| // All 16 first bytes of 3-byte sequences, each followed by a space |
| // (generates 16 invalid char + space sequences): |
| {{0xe0, 0x20, 0xe1, 0x20, 0xe2, 0x20, 0xe3, 0x20, 0xe4, 0x20, 0xe5, |
| 0x20, 0xe6, 0x20, 0xe7, 0x20, 0xe8, 0x20, 0xe9, 0x20, 0xea, 0x20, |
| 0xeb, 0x20, 0xec, 0x20, 0xed, 0x20, 0xee, 0x20, 0xef, 0x20}, |
| {0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20}}, |
| // All 8 first bytes of 4-byte sequences, each followed by a space |
| // (generates 8 invalid char + space sequences): |
| {{0xf0, 0x20, 0xf1, 0x20, 0xf2, 0x20, 0xf3, 0x20, 0xf4, 0x20, 0xf5, 0x20, |
| 0xf6, 0x20, 0xf7, 0x20}, |
| {0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, |
| 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20}}, |
| // All 4 first bytes of 5-byte sequences (not supported), each followed by |
| // a space (generates 4 invalid char + space sequences): |
| {{0xf8, 0x20, 0xf9, 0x20, 0xfa, 0x20, 0xfb, 0x20}, |
| {0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20, 0xfffd, 0x20}}, |
| // All 2 first bytes of 6-byte sequences (not supported), each followed by |
| // a space (generates 2 invalid char + space sequences): |
| {{0xfc, 0x20, 0xfd, 0x20}, {0xfffd, 0x20, 0xfffd, 0x20}}, |
| |
| // Sequences with last continuation byte missing. Normally the whole |
| // incomplete sequence generates a single invalid character (exceptions |
| // explained below). |
| |
| // 2-byte sequences with last byte missing |
| {{0xc0}, {0xfffd}}, |
| {{0xdf}, {0xfffd}}, |
| // 3-byte sequences with last byte missing. |
| {{0xe8, 0x80}, {0xfffd}}, |
| {{0xe0, 0xbf}, {0xfffd}}, |
| {{0xef, 0xbf}, {0xfffd}}, |
| // Start of an overlong sequence. The first "maximal subpart" is the first |
| // byte; it creates an invalid character. Each following byte generates an |
| // invalid character too. |
| {{0xe0, 0x80}, {0xfffd, 0xfffd}}, |
| // 4-byte sequences with last byte missing |
| {{0xf1, 0x80, 0x80}, {0xfffd}}, |
| {{0xf4, 0x8f, 0xbf}, {0xfffd}}, |
| // Start of an overlong sequence. The first "maximal subpart" is the first |
| // byte; it creates an invalid character. Each following byte generates an |
| // invalid character too. |
| {{0xf0, 0x80, 0x80}, {0xfffd, 0xfffd, 0xfffd}}, |
| // 5-byte sequences (not supported) with last byte missing |
| {{0xf8, 0x80, 0x80, 0x80}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xfb, 0xbf, 0xbf, 0xbf}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 6-byte sequences (not supported) with last byte missing |
| {{0xfc, 0x80, 0x80, 0x80, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xfd, 0xbf, 0xbf, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Concatenation of incomplete sequences: above incomplete sequences |
| // concatenated. |
| {{0xc0, 0xdf, 0xe8, 0x80, 0xe0, 0xbf, 0xef, 0xbf, 0xe0, 0x80, |
| 0xf1, 0x80, 0x80, 0xf4, 0x8f, 0xbf, 0xf0, 0x80, 0x80, 0xf8, |
| 0x80, 0x80, 0x80, 0xfb, 0xbf, 0xbf, 0xbf, 0xfc, 0x80, 0x80, |
| 0x80, 0x80, 0xfd, 0xbf, 0xbf, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, |
| 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Incomplete sequence tests repeated with a space after the incomplete |
| // sequence. |
| |
| // 2-byte sequences with last byte missing |
| {{0xc0, 0x20}, {0xfffd, 0x20}}, |
| {{0xdf, 0x20}, {0xfffd, 0x20}}, |
| // 3-byte sequences with last byte missing |
| {{0xe8, 0x80, 0x20}, {0xfffd, 0x20}}, |
| {{0xe0, 0xbf, 0x20}, {0xfffd, 0x20}}, |
| {{0xef, 0xbf, 0x20}, {0xfffd, 0x20}}, |
| // Start of overlong 3-byte sequence with last byte missing |
| {{0xe0, 0x80, 0x20}, {0xfffd, 0xfffd, 0x20}}, |
| // 4-byte sequences with last byte missing |
| {{0xf1, 0x80, 0x80, 0x20}, {0xfffd, 0x20}}, |
| {{0xf4, 0x8f, 0xbf, 0x20}, {0xfffd, 0x20}}, |
| // Start of overlong 4-byte sequence with last byte missing |
| {{0xf0, 0x80, 0x80, 0x20}, {0xfffd, 0xfffd, 0xfffd, 0x20}}, |
| // 5-byte sequences (not supported) with last byte missing |
| {{0xf8, 0x80, 0x80, 0x80, 0x20}, {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x20}}, |
| {{0xfb, 0xbf, 0xbf, 0xbf, 0x20}, {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x20}}, |
| // 6-byte sequences (not supported) with last byte missing |
| {{0xfc, 0x80, 0x80, 0x80, 0x80, 0x20}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x20}}, |
| {{0xfd, 0xbf, 0xbf, 0xbf, 0xbf, 0x20}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0x20}}, |
| |
| // Impossible bytes |
| {{0xfe}, {0xfffd}}, |
| {{0xff}, {0xfffd}}, |
| {{0xfe, 0xfe, 0xff, 0xff}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // Lead-byte-like bytes which aren't valid lead bytes. |
| {{0xc0}, {0xfffd}}, |
| {{0xc0, 0xaa}, {0xfffd, 0xfffd}}, |
| {{0xc1}, {0xfffd}}, |
| {{0xc1, 0xaa}, {0xfffd, 0xfffd}}, |
| {{0xf5}, {0xfffd}}, |
| {{0xf5, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xf6}, {0xfffd}}, |
| {{0xf6, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xf7}, {0xfffd}}, |
| {{0xf7, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xf8}, {0xfffd}}, |
| {{0xf8, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xf9}, {0xfffd}}, |
| {{0xf9, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xfa}, {0xfffd}}, |
| {{0xfa, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xfb}, {0xfffd}}, |
| {{0xfb, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xfc}, {0xfffd}}, |
| {{0xfc, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xfd}, {0xfffd}}, |
| {{0xfd, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xfe}, {0xfffd}}, |
| {{0xfe, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xff}, {0xfffd}}, |
| {{0xff, 0xaa, 0xaa, 0xaa}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Overlong sequences: |
| |
| // Overlong encodings for "/" |
| {{0xc0, 0xaf}, {0xfffd, 0xfffd}}, |
| {{0xe0, 0x80, 0xaf}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xf0, 0x80, 0x80, 0xaf}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 5-byte sequence (not supported anyway) |
| {{0xf8, 0x80, 0x80, 0x80, 0xaf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 6-byte sequence (not supported anyway) |
| {{0xfc, 0x80, 0x80, 0x80, 0x80, 0xaf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Maximum overlong sequences |
| {{0xc1, 0xbf}, {0xfffd, 0xfffd}}, |
| {{0xe0, 0x9f, 0xbf}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xf0, 0x8f, 0xbf, 0xbf}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 5-byte sequence (not supported anyway) |
| {{0xf8, 0x87, 0xbf, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 6-byte sequence (not supported anyway) |
| {{0xfc, 0x83, 0xbf, 0xbf, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Overlong encodings for 0 |
| {{0xc0, 0x80}, {0xfffd, 0xfffd}}, |
| {{0xe0, 0x80, 0x80}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xf0, 0x80, 0x80, 0x80}, {0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 5-byte sequence (not supported anyway) |
| {{0xf8, 0x80, 0x80, 0x80, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| // 6-byte sequence (not supported anyway) |
| {{0xfc, 0x80, 0x80, 0x80, 0x80, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Illegal code positions: |
| |
| // Single UTF-16 surrogates |
| {{0xed, 0xa0, 0x80}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xa0, 0x80}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xad, 0xbf}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xae, 0x80}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xaf, 0xbf}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xb0, 0x80}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xbe, 0x80}, {0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xbf, 0xbf}, {0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Paired surrogates |
| {{0xed, 0xa0, 0x80, 0xed, 0xb0, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xad, 0xbf, 0xed, 0xb0, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xad, 0xbf, 0xed, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xae, 0x80, 0xed, 0xb0, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xae, 0x80, 0xed, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xaf, 0xbf, 0xed, 0xb0, 0x80}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| {{0xed, 0xaf, 0xbf, 0xed, 0xbf, 0xbf}, |
| {0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd, 0xfffd}}, |
| |
| // Surrogates with the last byte missing. |
| {{0xed, 0xa0}, {0xfffd, 0xfffd}}, |
| {{0xed, 0xa0}, {0xfffd, 0xfffd}}, |
| {{0xed, 0xad}, {0xfffd, 0xfffd}}, |
| {{0xed, 0xae}, {0xfffd, 0xfffd}}, |
| {{0xed, 0xaf}, {0xfffd, 0xfffd}}, |
| {{0xed, 0xb0}, {0xfffd, 0xfffd}}, |
| {{0xed, 0xbe}, {0xfffd, 0xfffd}}, |
| {{0xed, 0xbf}, {0xfffd, 0xfffd}}, |
| |
| // Other non-characters |
| {{0xef, 0xbf, 0xbe}, {0xfffe}}, |
| {{0xef, 0xbf, 0xbf}, {0xffff}}, |
| {{0xef, 0xb7, 0x90, 0xef, 0xb7, 0x91, 0xef, 0xb7, 0x92, 0xef, 0xb7, 0x93, |
| 0xef, 0xb7, 0x94, 0xef, 0xb7, 0x95, 0xef, 0xb7, 0x96, 0xef, 0xb7, 0x97, |
| 0xef, 0xb7, 0x98, 0xef, 0xb7, 0x99, 0xef, 0xb7, 0x9a, 0xef, 0xb7, 0x9b, |
| 0xef, 0xb7, 0x9c, 0xef, 0xb7, 0x9d, 0xef, 0xb7, 0x9e, 0xef, 0xb7, 0x9f, |
| 0xef, 0xb7, 0xa0, 0xef, 0xb7, 0xa1, 0xef, 0xb7, 0xa2, 0xef, 0xb7, 0xa3, |
| 0xef, 0xb7, 0xa4, 0xef, 0xb7, 0xa5, 0xef, 0xb7, 0xa6, 0xef, 0xb7, 0xa7, |
| 0xef, 0xb7, 0xa8, 0xef, 0xb7, 0xa9, 0xef, 0xb7, 0xaa, 0xef, 0xb7, 0xab, |
| 0xef, 0xb7, 0xac, 0xef, 0xb7, 0xad, 0xef, 0xb7, 0xae, 0xef, 0xb7, 0xaf}, |
| {0xfdd0, 0xfdd1, 0xfdd2, 0xfdd3, 0xfdd4, 0xfdd5, 0xfdd6, 0xfdd7, |
| 0xfdd8, 0xfdd9, 0xfdda, 0xfddb, 0xfddc, 0xfddd, 0xfdde, 0xfddf, |
| 0xfde0, 0xfde1, 0xfde2, 0xfde3, 0xfde4, 0xfde5, 0xfde6, 0xfde7, |
| 0xfde8, 0xfde9, 0xfdea, 0xfdeb, 0xfdec, 0xfded, 0xfdee, 0xfdef}}, |
| {{0xf0, 0x9f, 0xbf, 0xbe, 0xf0, 0x9f, 0xbf, 0xbf, 0xf0, 0xaf, 0xbf, |
| 0xbe, 0xf0, 0xaf, 0xbf, 0xbf, 0xf0, 0xbf, 0xbf, 0xbe, 0xf0, 0xbf, |
| 0xbf, 0xbf, 0xf1, 0x8f, 0xbf, 0xbe, 0xf1, 0x8f, 0xbf, 0xbf, 0xf1, |
| 0x9f, 0xbf, 0xbe, 0xf1, 0x9f, 0xbf, 0xbf, 0xf1, 0xaf, 0xbf, 0xbe, |
| 0xf1, 0xaf, 0xbf, 0xbf, 0xf1, 0xbf, 0xbf, 0xbe, 0xf1, 0xbf, 0xbf, |
| 0xbf, 0xf2, 0x8f, 0xbf, 0xbe, 0xf2, 0x8f, 0xbf, 0xbf}, |
| {0x1fffe, 0x1ffff, 0x2fffe, 0x2ffff, 0x3fffe, 0x3ffff, 0x4fffe, 0x4ffff, |
| 0x5fffe, 0x5ffff, 0x6fffe, 0x6ffff, 0x7fffe, 0x7ffff, 0x8fffe, |
| 0x8ffff}}, |
| }; |
| |
| for (auto test : data) { |
| // For figuring out which test fails: |
| fprintf(stderr, "test: "); |
| for (auto b : test.bytes) { |
| fprintf(stderr, "%x ", b); |
| } |
| fprintf(stderr, "\n"); |
| |
| std::vector<unibrow::uchar> output_normal; |
| DecodeNormally(test.bytes, &output_normal); |
| |
| CHECK_EQ(output_normal.size(), test.unicode_expected.size()); |
| for (size_t i = 0; i < output_normal.size(); ++i) { |
| CHECK_EQ(output_normal[i], test.unicode_expected[i]); |
| } |
| |
| std::vector<unibrow::uchar> output_incremental; |
| DecodeIncrementally(test.bytes, &output_incremental); |
| |
| CHECK_EQ(output_incremental.size(), test.unicode_expected.size()); |
| for (size_t i = 0; i < output_incremental.size(); ++i) { |
| CHECK_EQ(output_incremental[i], test.unicode_expected[i]); |
| } |
| } |
| } |
| |
| } // namespace internal |
| } // namespace v8 |
