From 186369ff2bf88c9dab0404fc51db445a0d560e05 Mon Sep 17 00:00:00 2001 From: Greg Studer Date: Tue, 18 Oct 2016 14:21:04 -0400 Subject: [PATCH 1/3] Streaming JSON parser, helper for streaming arrays of small values --- .clang-format | 66 +++++++ include/jsoncpp/json/forward_buffer.h | 257 ++++++++++++++++++++++++++ include/jsoncpp/json/reader.h | 53 +++++- src/jsontestrunner/CMakeLists.txt | 2 +- src/jsontestrunner/main.cpp | 79 ++++++-- src/lib_json/json_reader.cpp | 245 +++++++++++++++++------- src/test_lib_json/CMakeLists.txt | 2 +- test/jsonarrstream/pass1.expected | 5 + test/jsonarrstream/pass1.json | 7 + test/jsonarrstream/pass2.expected | 3 + test/jsonarrstream/pass2.json | 6 + test/jsonarrstream/pass3.expected | 0 test/jsonarrstream/pass3.json | 4 + test/jsonarrstream/pass4.expected | 5 + test/jsonarrstream/pass4.json | 4 + test/jsonarrstream/pass5.expected | 76 ++++++++ test/jsonarrstream/pass5.json | 63 +++++++ test/jsonarrstream/pass6.expected | 6 + test/jsonarrstream/pass6.json | 4 + test/jsonarrstream/pass7.expected | 3 + test/jsonarrstream/pass7.json | 1 + test/jsonarrstream/pass8.expected | 4 + test/jsonarrstream/pass8.json | 1 + test/runjsontests.py | 51 +++-- test/rununittests.py | 4 + 25 files changed, 851 insertions(+), 100 deletions(-) create mode 100644 .clang-format create mode 100644 include/jsoncpp/json/forward_buffer.h create mode 100644 test/jsonarrstream/pass1.expected create mode 100644 test/jsonarrstream/pass1.json create mode 100644 test/jsonarrstream/pass2.expected create mode 100644 test/jsonarrstream/pass2.json create mode 100644 test/jsonarrstream/pass3.expected create mode 100644 test/jsonarrstream/pass3.json create mode 100644 test/jsonarrstream/pass4.expected create mode 100644 test/jsonarrstream/pass4.json create mode 100644 test/jsonarrstream/pass5.expected create mode 100644 test/jsonarrstream/pass5.json create mode 100644 test/jsonarrstream/pass6.expected create mode 100644 test/jsonarrstream/pass6.json create mode 100644 test/jsonarrstream/pass7.expected create mode 100644 test/jsonarrstream/pass7.json create mode 100644 test/jsonarrstream/pass8.expected create mode 100644 test/jsonarrstream/pass8.json diff --git a/.clang-format b/.clang-format new file mode 100644 index 0000000..cca7507 --- /dev/null +++ b/.clang-format @@ -0,0 +1,66 @@ +--- +Language: Cpp +# BasedOnStyle: Google +AccessModifierOffset: -4 +AlignAfterOpenBracket: true +AlignEscapedNewlinesLeft: true +AlignOperands: true +AlignTrailingComments: true +AllowAllParametersOfDeclarationOnNextLine: true +AllowShortBlocksOnASingleLine: false +AllowShortCaseLabelsOnASingleLine: false +AllowShortIfStatementsOnASingleLine: true +AllowShortLoopsOnASingleLine: true +AllowShortFunctionsOnASingleLine: All +AlwaysBreakAfterDefinitionReturnType: false +AlwaysBreakAfterReturnType: All +AlwaysBreakTemplateDeclarations: true +AlwaysBreakBeforeMultilineStrings: true +BreakBeforeBinaryOperators: None +BreakBeforeTernaryOperators: true +BreakConstructorInitializersBeforeComma: false +BinPackParameters: false +BinPackArguments: false +ColumnLimit: 80 +ConstructorInitializerAllOnOneLineOrOnePerLine: true +ConstructorInitializerIndentWidth: 4 +DerivePointerAlignment: false +ExperimentalAutoDetectBinPacking: false +IndentCaseLabels: true +IndentWrappedFunctionNames: false +IndentFunctionDeclarationAfterType: false +MaxEmptyLinesToKeep: 1 +KeepEmptyLinesAtTheStartOfBlocks: true +NamespaceIndentation: None +ObjCBlockIndentWidth: 2 +ObjCSpaceAfterProperty: false +ObjCSpaceBeforeProtocolList: false +PenaltyBreakBeforeFirstCallParameter: 1 +PenaltyBreakComment: 300 +PenaltyBreakString: 1000 +PenaltyBreakFirstLessLess: 120 +PenaltyExcessCharacter: 1000000 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Right +SpacesBeforeTrailingComments: 2 +Cpp11BracedListStyle: true +Standard: Auto +IndentWidth: 4 +TabWidth: 8 +UseTab: Never +BreakBeforeBraces: Allman +SpacesInParentheses: true +SpacesInSquareBrackets: false +SpacesInAngles: false +SpaceInEmptyParentheses: false +SpacesInCStyleCastParentheses: false +SpaceAfterCStyleCast: false +SpacesInContainerLiterals: true +SpaceBeforeAssignmentOperators: true +ContinuationIndentWidth: 4 +CommentPragmas: '^ IWYU pragma:' +ForEachMacros: [ foreach, Q_FOREACH, BOOST_FOREACH ] +SpaceBeforeParens: ControlStatements +DisableFormat: false +... + diff --git a/include/jsoncpp/json/forward_buffer.h b/include/jsoncpp/json/forward_buffer.h new file mode 100644 index 0000000..5bf3665 --- /dev/null +++ b/include/jsoncpp/json/forward_buffer.h @@ -0,0 +1,257 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace Json +{ + +/** + * Wraps a raw input stream into a buffered forward-iterator interface to + * allow parsing with backtracking. + * + * Iterators of this class mimic a raw character stream interface, with + * sensible implementations of pointer-arithmetic semantics. This allows + * it to slip underneath a string-based parsing implementation by acting + * like a contiguous character array of maximum size. + * + * A buffer may be initialized from a string or general input char stream. + */ +struct ForwardBuffer +{ + + /** + * Initialize a buffer from an input string. + */ + void + reset( const std::string &str_input ) + { + this->owned_stream.reset( new std::istringstream( str_input ) ); + reset( *( this->owned_stream ), str_input.size() ); + } + + /** + * Initialize a buffer from a character input stream. + */ + void + reset( std::istream &stream, + size_t input_size = std::numeric_limits::max() ) + { + reset( std::istreambuf_iterator( stream ), input_size ); + } + + /** + * Initialize a buffer from a one-pass character stream iterator. + */ + void + reset( std::istreambuf_iterator input, + size_t input_size = std::numeric_limits::max() ) + { + this->input = input; + this->input_size = input_size; + this->buffer.clear(); + } + + struct Iterator; + + /** + * Re-initialize a buffer, discarding data before a particular buffer + * position. + */ + void + resetTo( Iterator it ) + { + if ( it == end() ) + { + this->buffer.clear(); + this->input_size = 0; + return; + } + + this->buffer.erase( buffer.begin(), it.raw() ); + if ( this->input_size != std::numeric_limits::max() ) + this->input_size -= it.index; + } + + /** + * Iterator mimics a character pointer array of configurable size - for + * unbounded streams it is an array of maximum size. + */ + struct Iterator + { + + Iterator() : buffer( 0 ), index( 0 ) {} + + Iterator( int ptr ) + : buffer( reinterpret_cast( ptr ) ), index( 0 ) + { + } + + Iterator( ForwardBuffer *buffer, size_t index ) + : buffer( buffer ), index( index ) + { + } + + // ptr = 0 + Iterator & + operator=( int ptr ) + { + buffer = reinterpret_cast( ptr ); + return *this; + } + + // if (ptr) + operator bool() const { return buffer != 0; } + + // *ptr + const char &operator*() const { return buffer->buffer[index]; } + + // ++ptr + Iterator operator++() + { + index = buffer->nextIndex( index ); + return *this; + } + + // ptr++ + Iterator operator++( int ) + { + Iterator result( *this ); + ++( *this ); + return result; + } + + // ptr_a == ptr_b + bool + operator==( const Iterator &other ) const + { + return index == other.index; + } + + // ptr_a != ptr_b + bool + operator!=( const Iterator &other ) const + { + return !( *this == other ); + } + + // ptr_a < ptr_b + bool + operator<( const Iterator &other ) const + { + return index < other.index; + } + + // ptr - 1 + Iterator + operator-( int n ) const + { + return Iterator( buffer, index - n ); + } + + // ptr + 1 + Iterator + operator+( int n ) const + { + return Iterator( buffer, index + n ); + } + + // ptr += 1 + Iterator & + operator+=( int n ) + { + for ( int i = 0; i < n; ++i ) index = buffer->nextIndex( index ); + return *this; + } + + // ptr_a - ptr_b + size_t + operator-( const Iterator &other ) const + { + return index - other.index; + } + + // ptr[1] + const char &operator[]( size_t n ) const + { + return buffer->at( index + n ); + } + + /** + * Raw access to the buffer iterator, for std::string + */ + std::vector::iterator + raw() + { + return buffer->buffer.begin() + index; + } + + // We buffer sometimes in the background, but that shouldn't + // affect semantics + mutable ForwardBuffer *buffer; + size_t index; + }; + + Iterator + begin() + { + return Iterator( this, !bufferTo( 1 ) ? input_size : 0 ); + } + + Iterator + end() + { + return Iterator( this, input_size ); + } + + const char & + at( size_t index ) + { + bufferTo( index + 1 ); + return buffer[index]; + } + + bool + bufferTo( size_t size ) + { + while ( buffer.size() < size ) + { + if ( input == std::istreambuf_iterator() || + buffer.size() == input_size ) + { + return false; + } + buffer.push_back( *input ); + ++input; + } + + return true; + } + + /** + * Increment index, where last index is at input_size and may be + * non-contiguous. + */ + size_t + nextIndex( size_t index ) + { + if ( index == input_size ) return index; + return !bufferTo( ( index + 1 ) + 1 ) ? input_size : index + 1; + } + + std::string + getBuffered() + { + return std::string( buffer.begin(), buffer.end() ); + } + + // Sometimes it's convenient to own the stream + std::unique_ptr owned_stream; + std::istreambuf_iterator input; + size_t input_size; + std::vector buffer; +}; +} \ No newline at end of file diff --git a/include/jsoncpp/json/reader.h b/include/jsoncpp/json/reader.h index 189da57..4621065 100644 --- a/include/jsoncpp/json/reader.h +++ b/include/jsoncpp/json/reader.h @@ -9,6 +9,7 @@ #if !defined(JSON_IS_AMALGAMATION) # include "features.h" # include "value.h" +# include "forward_buffer.h" #endif // if !defined(JSON_IS_AMALGAMATION) # include # include @@ -29,8 +30,9 @@ namespace Json { class JSON_API Reader { public: + typedef char Char; - typedef const Char *Location; + typedef ForwardBuffer::Iterator Location; /** \brief Constructs a Reader allowing all features * for parsing. @@ -95,6 +97,9 @@ namespace Json { std::string getFormattedErrorMessages() const; private: + + friend class ArrayStreamReader; + enum TokenType { tokenEndOfStream = 0, @@ -131,10 +136,19 @@ namespace Json { typedef std::deque Errors; + void startParse( std::istream &sin, bool collectComments = true ); + void startParse( const std::string &document, bool collectComments = true ); + void restartParse( Location begin, bool collectComments = true ); + + bool readDocument( Value &root ); + + bool readNextArrayElement( Token &token, Json::Value &value ); + bool readNextValue( Json::Value &value ); + bool expectToken( TokenType type, Token &token, const char *message ); bool readToken( Token &token ); void skipSpaces(); - bool match( Location pattern, + bool match( const char* pattern, int patternLength ); bool readComment(); bool readCStyleComment(); @@ -179,6 +193,7 @@ namespace Json { Nodes nodes_; Errors errors_; std::string document_; + ForwardBuffer buffer_; Location begin_; Location end_; Location current_; @@ -213,7 +228,39 @@ namespace Json { \throw std::exception on parse error. \see Json::operator<<() */ - JSON_API std::istream& operator>>( std::istream&, Value& ); + JSON_API std::istream &operator>>( std::istream &, Value & ); + + /** + * A streaming reader of big JSON input files which consist of a root + * array with many element values. + * + * Reads one-element-at-a-time, which should give good performance when + * the elements themselves are small. + * + * Usage: + * ArrayStreamReader reader(stream); + * + * Json::Value value; + * bool error; + * while (parseNextElement(value, error)) { + * // Handle value + * } + * // Handle error + */ + class JSON_API ArrayStreamReader + { + public: + ArrayStreamReader( std::istream &sin ); + + ArrayStreamReader( const Features &features, std::istream &sin ); + + bool parseNextElement( Value &value, bool &error ); + + private: + std::istream &sin_; + Reader reader_; + Reader::Token token_; + }; } // namespace Json diff --git a/src/jsontestrunner/CMakeLists.txt b/src/jsontestrunner/CMakeLists.txt index b79236a..681561e 100644 --- a/src/jsontestrunner/CMakeLists.txt +++ b/src/jsontestrunner/CMakeLists.txt @@ -13,7 +13,7 @@ target_link_libraries(jsontestrunner jsoncpp) if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") set(test_path "PATH=$ENV{PATH};$") elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(test_path "DYLD_LIBRARY_PATH=$ENV{DYLD_LIBRARY_PATH}:$") + set(test_path "TEST_DYLD_LIBRARY_PATH=$ENV{DYLD_LIBRARY_PATH}:$") else() set(test_path "LD_LIBRARY_PATH=$:$ENV{DYLD_LIBRARY_PATH}") endif() diff --git a/src/jsontestrunner/main.cpp b/src/jsontestrunner/main.cpp index 74f0216..b3fdff8 100644 --- a/src/jsontestrunner/main.cpp +++ b/src/jsontestrunner/main.cpp @@ -10,6 +10,7 @@ #include #include // sort #include +#include #if defined(_MSC_VER) && _MSC_VER >= 1310 # pragma warning( disable: 4996 ) // disable fopen deprecation warning @@ -147,6 +148,35 @@ parseAndSaveValueTree( const std::string &input, return 0; } +static int +streamParseArrayElements( const std::string &input, + const std::string &actual, + const std::string &kind, + const Json::Features &features ) +{ + + std::istringstream input_stream( input ); + + Json::ArrayStreamReader reader( features, input_stream ); + + FILE *factual = fopen( actual.c_str(), "wt" ); + if ( !factual ) + { + printf( "Failed to create %s actual file.\n", kind.c_str() ); + return 2; + } + + Json::Value element; + bool error; + while ( reader.parseNextElement( element, error ) ) + { + printValueTree( factual, element ); + } + + fclose( factual ); + + return error ? 1 : 0; +} static int rewriteValueTree( const std::string &rewritePath, @@ -205,9 +235,12 @@ printUsage( const char *argv[] ) int parseCommandLine( int argc, const char *argv[], Json::Features &features, std::string &path, - bool &parseOnly ) + bool &parseOnly, + bool &streamParseArray ) { parseOnly = false; + streamParseArray = false; + if ( argc < 2 ) { return printUsage( argv ); @@ -227,6 +260,14 @@ parseCommandLine( int argc, const char *argv[], return 3; } + if ( std::string(argv[1]) == "--json-arr-stream" ) + { + features = Json::Features::all(); + parseOnly = true; + streamParseArray = true; + ++index; + } + if ( index == argc || index + 1 < argc ) { return printUsage( argv ); @@ -242,7 +283,10 @@ int main( int argc, const char *argv[] ) std::string path; Json::Features features; bool parseOnly; - int exitCode = parseCommandLine( argc, argv, features, path, parseOnly ); + bool streamParseArray; + int exitCode = parseCommandLine( argc, argv, features, path, + parseOnly, + streamParseArray ); if ( exitCode != 0 ) { return exitCode; @@ -257,7 +301,7 @@ int main( int argc, const char *argv[] ) return 3; } - std::string basePath = removeSuffix( argv[1], ".json" ); + std::string basePath = removeSuffix( path, ".json" ); if ( !parseOnly && basePath.empty() ) { printf( "Bad input path. Path does not end with '.expected':\n%s\n", path.c_str() ); @@ -269,17 +313,24 @@ int main( int argc, const char *argv[] ) std::string rewriteActualPath = basePath + ".actual-rewrite"; Json::Value root; - exitCode = parseAndSaveValueTree( input, actualPath, "input", root, features, parseOnly ); - if ( exitCode == 0 && !parseOnly ) - { - std::string rewrite; - exitCode = rewriteValueTree( rewritePath, root, rewrite ); - if ( exitCode == 0 ) - { - Json::Value rewriteRoot; - exitCode = parseAndSaveValueTree( rewrite, rewriteActualPath, - "rewrite", rewriteRoot, features, parseOnly ); - } + + if ( streamParseArray ) { + exitCode = streamParseArrayElements( input, actualPath, "input", features ); + } + else { + exitCode = parseAndSaveValueTree( input, actualPath, "input", root, features, parseOnly ); + + if ( exitCode == 0 && !parseOnly ) + { + std::string rewrite; + exitCode = rewriteValueTree( rewritePath, root, rewrite ); + if ( exitCode == 0 ) + { + Json::Value rewriteRoot; + exitCode = parseAndSaveValueTree( rewrite, rewriteActualPath, + "rewrite", rewriteRoot, features, parseOnly ); + } + } } } catch ( const std::exception &e ) diff --git a/src/lib_json/json_reader.cpp b/src/lib_json/json_reader.cpp index b1a742b..41c0c06 100644 --- a/src/lib_json/json_reader.cpp +++ b/src/lib_json/json_reader.cpp @@ -109,81 +109,141 @@ Reader::Reader( const Features &features ) { } +bool +Reader::parse( const std::string &document, Value &root, bool collectComments ) +{ + startParse( document, collectComments ); + return readDocument( root ); +} bool -Reader::parse( const std::string &document, +Reader::parse( const char *beginDoc, + const char *endDoc, Value &root, bool collectComments ) { - document_ = document; - const char *begin = document_.c_str(); - const char *end = begin + document_.length(); - return parse( begin, end, root, collectComments ); + return parse( std::string( beginDoc, endDoc ), root, collectComments ); } - bool -Reader::parse( std::istream& sin, - Value &root, - bool collectComments ) +Reader::parse( std::istream &sin, Value &root, bool collectComments ) { - //std::istream_iterator begin(sin); - //std::istream_iterator end; - // Those would allow streamed input from a file, if parse() were a - // template function. + startParse( sin, collectComments ); + return readDocument( root ); +} - // Since std::string is reference-counted, this at least does not - // create an extra copy. - //std::string doc; - //std::getline(sin, doc, std::ios::eof); - std::string doc((std::istreambuf_iterator(sin)), - std::istreambuf_iterator()); - return parse( doc, root, collectComments ); +void +Reader::startParse( const std::string &document, bool collectComments ) +{ + document_ = document; + buffer_.reset( document_ ); + restartParse( buffer_.begin(), collectComments ); } -bool -Reader::parse( const char *beginDoc, const char *endDoc, - Value &root, - bool collectComments ) +void +Reader::startParse( std::istream &sin, bool collectComments ) { - if ( !features_.allowComments_ ) - { - collectComments = false; - } + buffer_.reset( sin ); + restartParse( buffer_.begin(), collectComments ); +} - begin_ = beginDoc; - end_ = endDoc; - collectComments_ = collectComments; - current_ = begin_; - lastValueEnd_ = 0; - lastValue_ = 0; - commentsBefore_ = ""; - errors_.clear(); - while ( !nodes_.empty() ) - nodes_.pop(); - nodes_.push( &root ); - - bool successful = readValue(); - Token token; - skipCommentTokens( token ); - if ( collectComments_ && !commentsBefore_.empty() ) - root.setComment( commentsBefore_, commentAfter ); - if ( features_.strictRoot_ ) - { - if ( !root.isArray() && !root.isObject() ) - { - // Set error location to start of doc, ideally should be first token found in doc - token.type_ = tokenError; - token.start_ = beginDoc; - token.end_ = endDoc; - addError( "A valid JSON document must be either an array or an object value.", - token ); - return false; - } - } - return successful; +void +Reader::restartParse( Location start, bool collectComments ) +{ + buffer_.resetTo( start ); + begin_ = buffer_.begin(); + end_ = buffer_.end(); + current_ = begin_; + lastValueEnd_ = 0; + lastValue_ = 0; + + collectComments_ = ( features_.allowComments_ && collectComments ); + commentsBefore_ = ""; + + errors_.clear(); + while ( !nodes_.empty() ) nodes_.pop(); +} + +bool +Reader::readDocument( Value &root ) +{ + nodes_.push( &root ); + + bool successful = readValue(); + Token token; + skipCommentTokens( token ); + if ( collectComments_ && !commentsBefore_.empty() ) + root.setComment( commentsBefore_, commentAfter ); + if ( features_.strictRoot_ ) + { + if ( !root.isArray() && !root.isObject() ) + { + // Set error location to start of doc, ideally should be first token + // found in doc + token.type_ = tokenError; + token.start_ = begin_; + token.end_ = token.end_; + addError( + "A valid JSON document must be either an array or an object " + "value.", + token ); + return false; + } + } + return successful; +} + +bool +Reader::readNextArrayElement( Token &token, Json::Value &value ) +{ + if ( token.type_ != tokenArraySeparator ) + { + skipCommentTokens( token ); + if ( token.type_ != tokenArrayBegin ) + { + // Not the start of an array! + token.type_ = tokenError; + return false; + } + + skipSpaces(); + + if ( *current_ == ']' ) // empty array + { + Token endArray; + readToken( endArray ); + return false; + } + } + + // Read next array value + if ( !readNextValue( value ) ) + { + // Bad element value + token.type_ = tokenError; + return false; + } + + skipCommentTokens( token ); + + if ( token.type_ != tokenArraySeparator && token.type_ != tokenArrayEnd ) + { + // Array not continued or ended + token.type_ = tokenError; + } + + // ... but we've always got a valid value we read last + return true; } +bool +Reader::readNextValue( Json::Value &value ) +{ + nodes_.push( &value ); + bool ok = readValue(); + nodes_.pop(); + return ok; +} bool Reader::readValue() @@ -354,7 +414,7 @@ Reader::skipSpaces() bool -Reader::match( Location pattern, +Reader::match( const char* pattern, int patternLength ) { if ( end_ - current_ < patternLength ) @@ -401,17 +461,19 @@ Reader::addComment( Location begin, Location end, CommentPlacement placement ) { + std::string comment( begin.raw(), end.raw() ); + assert( collectComments_ ); if ( placement == commentAfterOnSameLine ) { assert( lastValue_ != 0 ); - lastValue_->setComment( std::string( begin, end ), placement ); + lastValue_->setComment( comment, placement ); } else { if ( !commentsBefore_.empty() ) commentsBefore_ += "\n"; - commentsBefore_ += std::string( begin, end ); + commentsBefore_ += comment; } } @@ -599,7 +661,7 @@ Reader::decodeNumber( Token &token ) { Char c = *current++; if ( c < '0' || c > '9' ) - return addError( "'" + std::string( token.start_, token.end_ ) + "' is not a number.", token ); + return addError( "'" + std::string( token.start_.raw(), token.end_.raw() ) + "' is not a number.", token ); Value::UInt digit(c - '0'); if ( value >= threshold ) { @@ -625,7 +687,6 @@ Reader::decodeNumber( Token &token ) return true; } - bool Reader::decodeDouble( Token &token ) { @@ -649,18 +710,20 @@ Reader::decodeDouble( Token &token ) if ( length <= bufferSize ) { Char buffer[bufferSize+1]; - memcpy( buffer, token.start_, length ); + Location it = token.start_; + for (int i = 0; i < length; ++i, ++it) + buffer[i] = *it; buffer[length] = 0; count = sscanf( buffer, format, &value ); } else { - std::string buffer( token.start_, token.end_ ); + std::string buffer( token.start_.raw(), token.end_.raw() ); count = sscanf( buffer.c_str(), format, &value ); } if ( count != 1 ) - return addError( "'" + std::string( token.start_, token.end_ ) + "' is not a number.", token ); + return addError( "'" + std::string( token.start_.raw(), token.end_.raw() ) + "' is not a number.", token ); currentValue() = value; return true; } @@ -918,5 +981,53 @@ std::istream& operator>>( std::istream &sin, Value &root ) return sin; } +ArrayStreamReader::ArrayStreamReader( std::istream &sin ) : sin_( sin ) +{ + token_.type_ = Reader::tokenEndOfStream; +} + +ArrayStreamReader::ArrayStreamReader( const Features &features, + std::istream &sin ) + : sin_( sin ), reader_( features ) +{ + token_.type_ = Reader::tokenEndOfStream; +} + +bool +ArrayStreamReader::parseNextElement( Value &value, bool &error ) +{ + if ( token_.type_ == Reader::tokenEndOfStream ) + { + // Sentinel start of parse + reader_.startParse( sin_ ); + } + else if ( token_.type_ == Reader::tokenArrayEnd ) + { + // Done with parse + return false; + } + else if ( token_.type_ != Reader::tokenArraySeparator ) + { + // Something weird happened + error = true; + return false; + } + + bool read_value = reader_.readNextArrayElement( token_, value ); + + if ( token_.type_ == Reader::tokenError ) + { + // Something weird happened + error = true; + } + else if ( token_.type_ == Reader::tokenArraySeparator || + token_.type_ == Reader::tokenArrayEnd ) + { + // Restart parsing at the end of the last read token + reader_.restartParse( token_.end_ ); + } + + return read_value; +} } // namespace Json diff --git a/src/test_lib_json/CMakeLists.txt b/src/test_lib_json/CMakeLists.txt index 2aa6b01..a85142a 100644 --- a/src/test_lib_json/CMakeLists.txt +++ b/src/test_lib_json/CMakeLists.txt @@ -12,7 +12,7 @@ target_link_libraries(test_lib_json jsoncpp) if(${CMAKE_SYSTEM_NAME} MATCHES "Windows") set(test_path "PATH=$ENV{PATH};$") elseif(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") - set(test_path "DYLD_LIBRARY_PATH=$ENV{DYLD_LIBRARY_PATH}:$") + set(test_path "TEST_DYLD_LIBRARY_PATH=$ENV{DYLD_LIBRARY_PATH}:$") else() set(test_path "LD_LIBRARY_PATH=$ENV{LD_LIBRARY_PATH}:$") endif() diff --git a/test/jsonarrstream/pass1.expected b/test/jsonarrstream/pass1.expected new file mode 100644 index 0000000..c83451d --- /dev/null +++ b/test/jsonarrstream/pass1.expected @@ -0,0 +1,5 @@ +.=1 +.=2 +.=3 +.=4 +.=5 diff --git a/test/jsonarrstream/pass1.json b/test/jsonarrstream/pass1.json new file mode 100644 index 0000000..2177d8a --- /dev/null +++ b/test/jsonarrstream/pass1.json @@ -0,0 +1,7 @@ +[ +1, +2, +3, +4, +5 +] diff --git a/test/jsonarrstream/pass2.expected b/test/jsonarrstream/pass2.expected new file mode 100644 index 0000000..9526959 --- /dev/null +++ b/test/jsonarrstream/pass2.expected @@ -0,0 +1,3 @@ +.=1 +.=2 +.=3 diff --git a/test/jsonarrstream/pass2.json b/test/jsonarrstream/pass2.json new file mode 100644 index 0000000..5046a29 --- /dev/null +++ b/test/jsonarrstream/pass2.json @@ -0,0 +1,6 @@ +[ // Comment +1 // Comment +, +2, // Comment +3 // Comment +] diff --git a/test/jsonarrstream/pass3.expected b/test/jsonarrstream/pass3.expected new file mode 100644 index 0000000..e69de29 diff --git a/test/jsonarrstream/pass3.json b/test/jsonarrstream/pass3.json new file mode 100644 index 0000000..e25b405 --- /dev/null +++ b/test/jsonarrstream/pass3.json @@ -0,0 +1,4 @@ +[ + + +] diff --git a/test/jsonarrstream/pass4.expected b/test/jsonarrstream/pass4.expected new file mode 100644 index 0000000..26c182b --- /dev/null +++ b/test/jsonarrstream/pass4.expected @@ -0,0 +1,5 @@ +.={} +.a=1 +.=[] +.[0]={} +.[0].b=2 diff --git a/test/jsonarrstream/pass4.json b/test/jsonarrstream/pass4.json new file mode 100644 index 0000000..3500b8f --- /dev/null +++ b/test/jsonarrstream/pass4.json @@ -0,0 +1,4 @@ +[ +{ "a" : 1}, +[ { "b" : 2 } ] +] diff --git a/test/jsonarrstream/pass5.expected b/test/jsonarrstream/pass5.expected new file mode 100644 index 0000000..12601e4 --- /dev/null +++ b/test/jsonarrstream/pass5.expected @@ -0,0 +1,76 @@ +.={} +.a=1 +.=[] +.[0]="JSON Test Pattern pass1" +.[1]={} +.[1].object with 1 member=[] +.[1].object with 1 member[0]="array with 1 element" +.[2]={} +.[3]=[] +.[4]=-42 +.[5]=true +.[6]=false +.[7]=null +.[8]={} +.[8].=2.3456789012e+76 +.[8]. s p a c e d =[] +.[8]. s p a c e d [0]=1 +.[8]. s p a c e d [1]=2 +.[8]. s p a c e d [2]=3 +.[8]. s p a c e d [3]=4 +.[8]. s p a c e d [4]=5 +.[8]. s p a c e d [5]=6 +.[8]. s p a c e d [6]=7 +.[8].# -- --> */=" " +.[8]./\"쫾몾ꮘﳞ볚 + `1~!@#$%^&*()_+-=[]{}|;:',./<>?="A key can be any string" +.[8].0123456789="digit" +.[8].ALPHA="ABCDEFGHIJKLMNOPQRSTUVWYZ" +.[8].E=1.23456789e+34 +.[8].address="50 St. James Street" +.[8].alpha="abcdefghijklmnopqrstuvwyz" +.[8].array=[] +.[8].backslash="\" +.[8].comment="// /* */": " ", + " s p a c e d " :[1,2 , 3 + +, + +4 , 5 , 6 ,7 ],"compact":[1,2,3,4,5,6,7], + "jsontext": "{\"object with 1 member\":[\"array with 1 element\"]}", + "quotes": "" \u0022 %22 0x22 034 "", + "\/\\\"\uCAFE\uBABE\uAB98\uFCDE\ubcda\uef4A\b\f\n\r\t`1~!@#$%^&*()_+-=[]{}|;:',./<>?" +: "A key can be any string" + }, + 0.5 ,98.6 +, +99.44 +, + +1066, +1e1, +0.1e1, +1e-1, +1e00,2e+00,2e-00 +,"rosebud"], +[], +{"c" : 3} +] diff --git a/test/jsonarrstream/pass6.expected b/test/jsonarrstream/pass6.expected new file mode 100644 index 0000000..3b9fa28 --- /dev/null +++ b/test/jsonarrstream/pass6.expected @@ -0,0 +1,6 @@ +.={} +.=[] +.={} +.=[] +.={} +.=[] diff --git a/test/jsonarrstream/pass6.json b/test/jsonarrstream/pass6.json new file mode 100644 index 0000000..185a998 --- /dev/null +++ b/test/jsonarrstream/pass6.json @@ -0,0 +1,4 @@ +[{ "" : [] } +,{ "" : [] } +,{ "" : [] } +] diff --git a/test/jsonarrstream/pass7.expected b/test/jsonarrstream/pass7.expected new file mode 100644 index 0000000..4dae681 --- /dev/null +++ b/test/jsonarrstream/pass7.expected @@ -0,0 +1,3 @@ +.={} +.={} +.={} diff --git a/test/jsonarrstream/pass7.json b/test/jsonarrstream/pass7.json new file mode 100644 index 0000000..069e030 --- /dev/null +++ b/test/jsonarrstream/pass7.json @@ -0,0 +1 @@ +[{},{},{}] diff --git a/test/jsonarrstream/pass8.expected b/test/jsonarrstream/pass8.expected new file mode 100644 index 0000000..6219352 --- /dev/null +++ b/test/jsonarrstream/pass8.expected @@ -0,0 +1,4 @@ +.="" +.="" +.=1 +.=0.55 diff --git a/test/jsonarrstream/pass8.json b/test/jsonarrstream/pass8.json new file mode 100644 index 0000000..c47aa5c --- /dev/null +++ b/test/jsonarrstream/pass8.json @@ -0,0 +1 @@ +["","",1,0.55] diff --git a/test/runjsontests.py b/test/runjsontests.py index 9689366..6f38ae5 100644 --- a/test/runjsontests.py +++ b/test/runjsontests.py @@ -7,6 +7,10 @@ VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes ' +# Hack for OSX el capitan and above +if "TEST_DYLD_LIBRARY_PATH" in os.environ: + os.environ["DYLD_LIBRARY_PATH"] = os.environ["TEST_DYLD_LIBRARY_PATH"] + def compareOutputs( expected, actual, message ): expected = expected.strip().replace('\r','').split('\n') actual = actual.strip().replace('\r','').split('\n') @@ -39,17 +43,25 @@ def safeReadFile( path ): return '' % (path,e) def runAllTests( jsontest_executable_path, input_dir = None, - use_valgrind=False, with_json_checker=False ): + use_valgrind=False, with_json_checker=False, + with_json_arr_stream=True ): if not input_dir: input_dir = os.path.join( os.getcwd(), 'data' ) tests = glob( os.path.join( input_dir, '*.json' ) ) + if with_json_checker: test_jsonchecker = glob( os.path.join( input_dir, '../jsonchecker', '*.json' ) ) else: test_jsonchecker = [] + + if with_json_arr_stream: + test_json_arr_stream = glob( os.path.join( input_dir, '../jsonarrstream', '*.json' ) ) + else: + test_json_arr_stream = [] + failed_tests = [] - for input_path in tests + test_jsonchecker: + for input_path in tests + test_jsonchecker + test_json_arr_stream: cmd = [] if use_valgrind: cmd.append(VALGRIND_CMD) @@ -57,18 +69,24 @@ def runAllTests( jsontest_executable_path, input_dir = None, cmd.append(jsontest_executable_path) is_json_checker_test = input_path in test_jsonchecker + is_json_arr_stream_test = input_path in test_json_arr_stream if is_json_checker_test: cmd.append('--json-checker') + elif is_json_arr_stream_test: + cmd.append('--json-arr-stream') cmd.append(input_path) print 'TESTING:', cmd - try: - process_output = subprocess.check_output(cmd) - status = None - except subprocess.CalledProcessError as e: - status = e.returncode + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + process_output, process_err = process.communicate() + status = None if process.returncode == 0 else process.returncode + + if status: + process_output = "(stdout)\n" + process_output + \ + "\n(stderr)\n" + process_err if is_json_checker_test: expect_failure = os.path.basename( input_path ).startswith( 'fail' ) @@ -95,14 +113,19 @@ def runAllTests( jsontest_executable_path, input_dir = None, failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) ) else: expected_output_path = os.path.splitext(input_path)[0] + '.expected' - expected_output = file( expected_output_path, 'rt' ).read() - detail = ( compareOutputs( expected_output, actual_output, 'input' ) - or compareOutputs( expected_output, actual_rewrite_output, 'rewrite' ) ) - if detail: - print 'FAILED' - failed_tests.append( (input_path, detail) ) + + if not os.path.exists( expected_output_path ): + print 'PARSE ONLY' else: - print 'OK' + expected_output = file( expected_output_path, 'rt' ).read() + detail = compareOutputs( expected_output, actual_output, 'input' ) + if not detail and os.path.exists(base_path + '.actual-rewrite'): + detail = compareOutputs( expected_output, actual_rewrite_output, 'rewrite' ) + if detail: + print 'FAILED' + failed_tests.append( (input_path, detail) ) + else: + print 'OK' if failed_tests: print diff --git a/test/rununittests.py b/test/rununittests.py index ccc54e4..bf3bfdc 100644 --- a/test/rununittests.py +++ b/test/rununittests.py @@ -7,6 +7,10 @@ VALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes' +# Hack for OSX el capitan and above +if "TEST_DYLD_LIBRARY_PATH" in os.environ: + os.environ["DYLD_LIBRARY_PATH"] = os.environ["TEST_DYLD_LIBRARY_PATH"] + class TestProxy(object): def __init__( self, test_exe_path, use_valgrind=False ): self.test_exe_path = os.path.normpath( os.path.abspath( test_exe_path ) ) From e59b73f9fcf9ed8762ef02218e586dc311911ed5 Mon Sep 17 00:00:00 2001 From: Greg Studer Date: Thu, 20 Oct 2016 16:48:24 -0400 Subject: [PATCH 2/3] add limits include for compile --- include/jsoncpp/json/forward_buffer.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/jsoncpp/json/forward_buffer.h b/include/jsoncpp/json/forward_buffer.h index 5bf3665..a90be80 100644 --- a/include/jsoncpp/json/forward_buffer.h +++ b/include/jsoncpp/json/forward_buffer.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace Json { From 513f3bcdc35ecc4193febabb769bbb731f0c300f Mon Sep 17 00:00:00 2001 From: Greg Studer Date: Fri, 21 Oct 2016 09:21:07 -0400 Subject: [PATCH 3/3] always initialize error value to something sane --- src/lib_json/json_reader.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib_json/json_reader.cpp b/src/lib_json/json_reader.cpp index 41c0c06..33092d7 100644 --- a/src/lib_json/json_reader.cpp +++ b/src/lib_json/json_reader.cpp @@ -996,6 +996,7 @@ ArrayStreamReader::ArrayStreamReader( const Features &features, bool ArrayStreamReader::parseNextElement( Value &value, bool &error ) { + error = false; if ( token_.type_ == Reader::tokenEndOfStream ) { // Sentinel start of parse