| #region Copyright notice and license |
| // Protocol Buffers - Google's data interchange format |
| // Copyright 2008 Google Inc. All rights reserved. |
| // https://developers.google.com/protocol-buffers/ |
| // |
| // Redistribution and use in source and binary forms, with or without |
| // modification, are permitted provided that the following conditions are |
| // met: |
| // |
| // * Redistributions of source code must retain the above copyright |
| // notice, this list of conditions and the following disclaimer. |
| // * Redistributions in binary form must reproduce the above |
| // copyright notice, this list of conditions and the following disclaimer |
| // in the documentation and/or other materials provided with the |
| // distribution. |
| // * Neither the name of Google Inc. nor the names of its |
| // contributors may be used to endorse or promote products derived from |
| // this software without specific prior written permission. |
| // |
| // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS |
| // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT |
| // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR |
| // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT |
| // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
| // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
| // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| #endregion |
| using NUnit.Framework; |
| using System; |
| using System.IO; |
| |
| namespace Google.Protobuf |
| { |
| public class JsonTokenizerTest |
| { |
| [Test] |
| public void EmptyObjectValue() |
| { |
| AssertTokens("{}", JsonToken.StartObject, JsonToken.EndObject); |
| } |
| |
| [Test] |
| public void EmptyArrayValue() |
| { |
| AssertTokens("[]", JsonToken.StartArray, JsonToken.EndArray); |
| } |
| |
| [Test] |
| [TestCase("foo", "foo")] |
| [TestCase("tab\\t", "tab\t")] |
| [TestCase("line\\nfeed", "line\nfeed")] |
| [TestCase("carriage\\rreturn", "carriage\rreturn")] |
| [TestCase("back\\bspace", "back\bspace")] |
| [TestCase("form\\ffeed", "form\ffeed")] |
| [TestCase("escaped\\/slash", "escaped/slash")] |
| [TestCase("escaped\\\\backslash", "escaped\\backslash")] |
| [TestCase("escaped\\\"quote", "escaped\"quote")] |
| [TestCase("foo {}[] bar", "foo {}[] bar")] |
| [TestCase("foo\\u09aFbar", "foo\u09afbar")] // Digits, upper hex, lower hex |
| [TestCase("ab\ud800\udc00cd", "ab\ud800\udc00cd")] |
| [TestCase("ab\\ud800\\udc00cd", "ab\ud800\udc00cd")] |
| public void StringValue(string json, string expectedValue) |
| { |
| AssertTokensNoReplacement("\"" + json + "\"", JsonToken.Value(expectedValue)); |
| } |
| |
| // Valid surrogate pairs, with mixed escaping. These test cases can't be expressed |
| // using TestCase as they have no valid UTF-8 representation. |
| // It's unclear exactly how we should handle a mixture of escaped or not: that can't |
| // come from UTF-8 text, but could come from a .NET string. For the moment, |
| // treat it as valid in the obvious way. |
| [Test] |
| public void MixedSurrogatePairs() |
| { |
| string expected = "\ud800\udc00"; |
| AssertTokens("'\\ud800\udc00'", JsonToken.Value(expected)); |
| AssertTokens("'\ud800\\udc00'", JsonToken.Value(expected)); |
| } |
| |
| [Test] |
| public void ObjectDepth() |
| { |
| string json = "{ \"foo\": { \"x\": 1, \"y\": [ 0 ] } }"; |
| var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); |
| // If we had more tests like this, I'd introduce a helper method... but for one test, it's not worth it. |
| Assert.AreEqual(0, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); |
| Assert.AreEqual(1, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.Name("foo"), tokenizer.Next()); |
| Assert.AreEqual(1, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.StartObject, tokenizer.Next()); |
| Assert.AreEqual(2, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.Name("x"), tokenizer.Next()); |
| Assert.AreEqual(2, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.Value(1), tokenizer.Next()); |
| Assert.AreEqual(2, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.Name("y"), tokenizer.Next()); |
| Assert.AreEqual(2, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.StartArray, tokenizer.Next()); |
| Assert.AreEqual(2, tokenizer.ObjectDepth); // Depth hasn't changed in array |
| Assert.AreEqual(JsonToken.Value(0), tokenizer.Next()); |
| Assert.AreEqual(2, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.EndArray, tokenizer.Next()); |
| Assert.AreEqual(2, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.EndObject, tokenizer.Next()); |
| Assert.AreEqual(1, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.EndObject, tokenizer.Next()); |
| Assert.AreEqual(0, tokenizer.ObjectDepth); |
| Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); |
| Assert.AreEqual(0, tokenizer.ObjectDepth); |
| } |
| |
| [Test] |
| public void ObjectDepth_WithPushBack() |
| { |
| string json = "{}"; |
| var tokenizer = JsonTokenizer.FromTextReader(new StringReader(json)); |
| Assert.AreEqual(0, tokenizer.ObjectDepth); |
| var token = tokenizer.Next(); |
| Assert.AreEqual(1, tokenizer.ObjectDepth); |
| // When we push back a "start object", we should effectively be back to the previous depth. |
| tokenizer.PushBack(token); |
| Assert.AreEqual(0, tokenizer.ObjectDepth); |
| // Read the same token again, and get back to depth 1 |
| token = tokenizer.Next(); |
| Assert.AreEqual(1, tokenizer.ObjectDepth); |
| |
| // Now the same in reverse, with EndObject |
| token = tokenizer.Next(); |
| Assert.AreEqual(0, tokenizer.ObjectDepth); |
| tokenizer.PushBack(token); |
| Assert.AreEqual(1, tokenizer.ObjectDepth); |
| tokenizer.Next(); |
| Assert.AreEqual(0, tokenizer.ObjectDepth); |
| } |
| |
| [Test] |
| [TestCase("embedded tab\t")] |
| [TestCase("embedded CR\r")] |
| [TestCase("embedded LF\n")] |
| [TestCase("embedded bell\u0007")] |
| [TestCase("bad escape\\a")] |
| [TestCase("incomplete escape\\")] |
| [TestCase("incomplete Unicode escape\\u000")] |
| [TestCase("invalid Unicode escape\\u000H")] |
| // Surrogate pair handling, both in raw .NET strings and escaped. We only need |
| // to detect this in strings, as non-ASCII characters anywhere other than in strings |
| // will already lead to parsing errors. |
| [TestCase("\\ud800")] |
| [TestCase("\\udc00")] |
| [TestCase("\\ud800x")] |
| [TestCase("\\udc00x")] |
| [TestCase("\\udc00\\ud800y")] |
| public void InvalidStringValue(string json) |
| { |
| AssertThrowsAfter("\"" + json + "\""); |
| } |
| |
| // Tests for invalid strings that can't be expressed in attributes, |
| // as the constants can't be expressed as UTF-8 strings. |
| [Test] |
| public void InvalidSurrogatePairs() |
| { |
| AssertThrowsAfter("\"\ud800x\""); |
| AssertThrowsAfter("\"\udc00y\""); |
| AssertThrowsAfter("\"\udc00\ud800y\""); |
| } |
| |
| [Test] |
| [TestCase("0", 0)] |
| [TestCase("-0", 0)] // We don't distinguish between positive and negative 0 |
| [TestCase("1", 1)] |
| [TestCase("-1", -1)] |
| // From here on, assume leading sign is okay... |
| [TestCase("1.125", 1.125)] |
| [TestCase("1.0", 1)] |
| [TestCase("1e5", 100000)] |
| [TestCase("1e000000", 1)] // Weird, but not prohibited by the spec |
| [TestCase("1E5", 100000)] |
| [TestCase("1e+5", 100000)] |
| [TestCase("1E-5", 0.00001)] |
| [TestCase("123E-2", 1.23)] |
| [TestCase("123.45E3", 123450)] |
| [TestCase(" 1 ", 1)] |
| public void NumberValue(string json, double expectedValue) |
| { |
| AssertTokens(json, JsonToken.Value(expectedValue)); |
| } |
| |
| [Test] |
| [TestCase("00")] |
| [TestCase(".5")] |
| [TestCase("1.")] |
| [TestCase("1e")] |
| [TestCase("1e-")] |
| [TestCase("--")] |
| [TestCase("--1")] |
| [TestCase("-1.7977e308")] |
| [TestCase("1.7977e308")] |
| public void InvalidNumberValue(string json) |
| { |
| AssertThrowsAfter(json); |
| } |
| |
| [Test] |
| [TestCase("nul")] |
| [TestCase("nothing")] |
| [TestCase("truth")] |
| [TestCase("fALSEhood")] |
| public void InvalidLiterals(string json) |
| { |
| AssertThrowsAfter(json); |
| } |
| |
| [Test] |
| public void NullValue() |
| { |
| AssertTokens("null", JsonToken.Null); |
| } |
| |
| [Test] |
| public void TrueValue() |
| { |
| AssertTokens("true", JsonToken.True); |
| } |
| |
| [Test] |
| public void FalseValue() |
| { |
| AssertTokens("false", JsonToken.False); |
| } |
| |
| [Test] |
| public void SimpleObject() |
| { |
| AssertTokens("{'x': 'y'}", |
| JsonToken.StartObject, JsonToken.Name("x"), JsonToken.Value("y"), JsonToken.EndObject); |
| } |
| |
| [Test] |
| [TestCase("[10, 20", 3)] |
| [TestCase("[10,", 2)] |
| [TestCase("[10:20]", 2)] |
| [TestCase("[", 1)] |
| [TestCase("[,", 1)] |
| [TestCase("{", 1)] |
| [TestCase("{,", 1)] |
| [TestCase("{[", 1)] |
| [TestCase("{{", 1)] |
| [TestCase("{0", 1)] |
| [TestCase("{null", 1)] |
| [TestCase("{false", 1)] |
| [TestCase("{true", 1)] |
| [TestCase("}", 0)] |
| [TestCase("]", 0)] |
| [TestCase(",", 0)] |
| [TestCase("'foo' 'bar'", 1)] |
| [TestCase(":", 0)] |
| [TestCase("'foo", 0)] // Incomplete string |
| [TestCase("{ 'foo' }", 2)] |
| [TestCase("{ x:1", 1)] // Property names must be quoted |
| [TestCase("{]", 1)] |
| [TestCase("[}", 1)] |
| [TestCase("[1,", 2)] |
| [TestCase("{'x':0]", 3)] |
| [TestCase("{ 'foo': }", 2)] |
| [TestCase("{ 'foo':'bar', }", 3)] |
| public void InvalidStructure(string json, int expectedValidTokens) |
| { |
| // Note: we don't test that the earlier tokens are exactly as expected, |
| // partly because that's hard to parameterize. |
| var reader = new StringReader(json.Replace('\'', '"')); |
| var tokenizer = JsonTokenizer.FromTextReader(reader); |
| for (int i = 0; i < expectedValidTokens; i++) |
| { |
| Assert.IsNotNull(tokenizer.Next()); |
| } |
| Assert.Throws<InvalidJsonException>(() => tokenizer.Next()); |
| } |
| |
| [Test] |
| public void ArrayMixedType() |
| { |
| AssertTokens("[1, 'foo', null, false, true, [2], {'x':'y' }]", |
| JsonToken.StartArray, |
| JsonToken.Value(1), |
| JsonToken.Value("foo"), |
| JsonToken.Null, |
| JsonToken.False, |
| JsonToken.True, |
| JsonToken.StartArray, |
| JsonToken.Value(2), |
| JsonToken.EndArray, |
| JsonToken.StartObject, |
| JsonToken.Name("x"), |
| JsonToken.Value("y"), |
| JsonToken.EndObject, |
| JsonToken.EndArray); |
| } |
| |
| [Test] |
| public void ObjectMixedType() |
| { |
| AssertTokens(@"{'a': 1, 'b': 'bar', 'c': null, 'd': false, 'e': true, |
| 'f': [2], 'g': {'x':'y' }}", |
| JsonToken.StartObject, |
| JsonToken.Name("a"), |
| JsonToken.Value(1), |
| JsonToken.Name("b"), |
| JsonToken.Value("bar"), |
| JsonToken.Name("c"), |
| JsonToken.Null, |
| JsonToken.Name("d"), |
| JsonToken.False, |
| JsonToken.Name("e"), |
| JsonToken.True, |
| JsonToken.Name("f"), |
| JsonToken.StartArray, |
| JsonToken.Value(2), |
| JsonToken.EndArray, |
| JsonToken.Name("g"), |
| JsonToken.StartObject, |
| JsonToken.Name("x"), |
| JsonToken.Value("y"), |
| JsonToken.EndObject, |
| JsonToken.EndObject); |
| } |
| |
| [Test] |
| public void NextAfterEndDocumentThrows() |
| { |
| var tokenizer = JsonTokenizer.FromTextReader(new StringReader("null")); |
| Assert.AreEqual(JsonToken.Null, tokenizer.Next()); |
| Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); |
| Assert.Throws<InvalidOperationException>(() => tokenizer.Next()); |
| } |
| |
| [Test] |
| public void CanPushBackEndDocument() |
| { |
| var tokenizer = JsonTokenizer.FromTextReader(new StringReader("null")); |
| Assert.AreEqual(JsonToken.Null, tokenizer.Next()); |
| Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); |
| tokenizer.PushBack(JsonToken.EndDocument); |
| Assert.AreEqual(JsonToken.EndDocument, tokenizer.Next()); |
| Assert.Throws<InvalidOperationException>(() => tokenizer.Next()); |
| } |
| |
| /// <summary> |
| /// Asserts that the specified JSON is tokenized into the given sequence of tokens. |
| /// All apostrophes are first converted to double quotes, allowing any tests |
| /// that don't need to check actual apostrophe handling to use apostrophes in the JSON, avoiding |
| /// messy string literal escaping. The "end document" token is not specified in the list of |
| /// expected tokens, but is implicit. |
| /// </summary> |
| private static void AssertTokens(string json, params JsonToken[] expectedTokens) |
| { |
| AssertTokensNoReplacement(json.Replace('\'', '"'), expectedTokens); |
| } |
| |
| /// <summary> |
| /// Asserts that the specified JSON is tokenized into the given sequence of tokens. |
| /// Unlike <see cref="AssertTokens(string, JsonToken[])"/>, this does not perform any character |
| /// replacement on the specified JSON, and should be used when the text contains apostrophes which |
| /// are expected to be used *as* apostrophes. The "end document" token is not specified in the list of |
| /// expected tokens, but is implicit. |
| /// </summary> |
| private static void AssertTokensNoReplacement(string json, params JsonToken[] expectedTokens) |
| { |
| var reader = new StringReader(json); |
| var tokenizer = JsonTokenizer.FromTextReader(reader); |
| for (int i = 0; i < expectedTokens.Length; i++) |
| { |
| var actualToken = tokenizer.Next(); |
| if (actualToken == JsonToken.EndDocument) |
| { |
| Assert.Fail("Expected {0} but reached end of token stream", expectedTokens[i]); |
| } |
| Assert.AreEqual(expectedTokens[i], actualToken); |
| } |
| var finalToken = tokenizer.Next(); |
| if (finalToken != JsonToken.EndDocument) |
| { |
| Assert.Fail("Expected token stream to be exhausted; received {0}", finalToken); |
| } |
| } |
| |
| private static void AssertThrowsAfter(string json, params JsonToken[] expectedTokens) |
| { |
| var reader = new StringReader(json); |
| var tokenizer = JsonTokenizer.FromTextReader(reader); |
| for (int i = 0; i < expectedTokens.Length; i++) |
| { |
| var actualToken = tokenizer.Next(); |
| if (actualToken == JsonToken.EndDocument) |
| { |
| Assert.Fail("Expected {0} but reached end of document", expectedTokens[i]); |
| } |
| Assert.AreEqual(expectedTokens[i], actualToken); |
| } |
| Assert.Throws<InvalidJsonException>(() => tokenizer.Next()); |
| } |
| } |
| } |