#include "yajl_common.h"

Include dependency graph for yajl_lex.h:

This graph shows which files directly or indirectly include this file:

Typedefs
typedef struct yajl_lexer_t *	yajl_lexer

Functions
yajl_lexer	yajl_lex_alloc (yajl_alloc_funcs *alloc, unsigned int allowComments, unsigned int validateUTF8)

void	yajl_lex_free (yajl_lexer lexer)

yajl_tok	yajl_lex_lex (yajl_lexer lexer, const unsigned char jsonText, size_t jsonTextLen, size_t offset, const unsigned char *outBuf, size_t outLen)

yajl_tok	yajl_lex_peek (yajl_lexer lexer, const unsigned char *jsonText, size_t jsonTextLen, size_t offset)

const char *	yajl_lex_error_to_string (yajl_lex_error error)

yajl_lex_error	yajl_lex_get_error (yajl_lexer lexer)

size_t	yajl_lex_current_offset (yajl_lexer lexer)

size_t	yajl_lex_current_line (yajl_lexer lexer)

size_t	yajl_lex_current_char (yajl_lexer lexer)

Typedef Documentation

typedef struct yajl_lexer_t* yajl_lexer

Definition at line 48 of file yajl_lex.h.

Enumeration Type Documentation

enum yajl_lex_error

Enumerator
yajl_lex_e_ok
yajl_lex_string_invalid_utf8
yajl_lex_string_invalid_escaped_char
yajl_lex_string_invalid_json_char
yajl_lex_string_invalid_hex_char
yajl_lex_invalid_char
yajl_lex_invalid_string
yajl_lex_missing_integer_after_decimal
yajl_lex_missing_integer_after_exponent
yajl_lex_missing_integer_after_minus
yajl_lex_unallowed_comment

Definition at line 91 of file yajl_lex.h.

              {
     yajl_lex_e_ok = 0,
     yajl_lex_string_invalid_utf8,
     yajl_lex_string_invalid_escaped_char,
     yajl_lex_string_invalid_json_char,
     yajl_lex_string_invalid_hex_char,
     yajl_lex_invalid_char,
     yajl_lex_invalid_string,
     yajl_lex_missing_integer_after_decimal,
     yajl_lex_missing_integer_after_exponent,
     yajl_lex_missing_integer_after_minus,
     yajl_lex_unallowed_comment
 } yajl_lex_error;

enum yajl_tok

Enumerator
yajl_tok_bool
yajl_tok_colon
yajl_tok_comma
yajl_tok_eof
yajl_tok_error
yajl_tok_left_brace
yajl_tok_left_bracket
yajl_tok_null
yajl_tok_right_brace
yajl_tok_right_bracket
yajl_tok_integer
yajl_tok_double
yajl_tok_string
yajl_tok_string_with_escapes
yajl_tok_comment

Definition at line 22 of file yajl_lex.h.

              {
     yajl_tok_bool,
     yajl_tok_colon,
     yajl_tok_comma,
     yajl_tok_eof,
     yajl_tok_error,
     yajl_tok_left_brace,
     yajl_tok_left_bracket,
     yajl_tok_null,
     yajl_tok_right_brace,
     yajl_tok_right_bracket,
 
     /* we differentiate between integers and doubles to allow the
      * parser to interpret the number without re-scanning */
     yajl_tok_integer,
     yajl_tok_double,
 
     /* we differentiate between strings which require further processing,
      * and strings that do not */
     yajl_tok_string,
     yajl_tok_string_with_escapes,
 
     /* comment tokens are not currently returned to the parser, ever */
     yajl_tok_comment
 } yajl_tok;

Function Documentation

yajl_lexer yajl_lex_alloc	(	yajl_alloc_funcs *	alloc,
		unsigned int	allowComments,
		unsigned int	validateUTF8
	)

Definition at line 104 of file yajl_lex.c.

 {
     yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t));
     if (lxr == NULL) {
         return NULL;
     }
 
     memset((void *) lxr, 0, sizeof(struct yajl_lexer_t));
     lxr->buf = yajl_buf_alloc(alloc);
     lxr->allowComments = allowComments;
     lxr->validateUTF8 = validateUTF8;
     lxr->alloc = alloc;
     return lxr;
 }

size_t yajl_lex_current_char ( yajl_lexer lexer )

get the number of chars lexed by this lexer instance since the last
or

Definition at line 744 of file yajl_lex.c.

 {
     return lexer->charOff;
 }

size_t yajl_lex_current_line ( yajl_lexer lexer )

get the number of lines lexed by this lexer instance

Definition at line 739 of file yajl_lex.c.

 {
     return lexer->lineOff;
 }

size_t yajl_lex_current_offset ( yajl_lexer lexer )

get the current offset into the most recently lexed json string.

const char* yajl_lex_error_to_string ( yajl_lex_error error )

Definition at line 695 of file yajl_lex.c.

 {
     switch (error) {
         case yajl_lex_e_ok:
             return "ok, no error";
         case yajl_lex_string_invalid_utf8:
             return "invalid bytes in UTF8 string.";
         case yajl_lex_string_invalid_escaped_char:
             return "inside a string, '\\' occurs before a character "
                    "which it may not.";
         case yajl_lex_string_invalid_json_char:
             return "invalid character inside string.";
         case yajl_lex_string_invalid_hex_char:
             return "invalid (non-hex) character occurs after '\\u' inside "
                    "string.";
         case yajl_lex_invalid_char:
             return "invalid char in json text.";
         case yajl_lex_invalid_string:
             return "invalid string in json text.";
         case yajl_lex_missing_integer_after_exponent:
             return "malformed number, a digit is required after the exponent.";
         case yajl_lex_missing_integer_after_decimal:
             return "malformed number, a digit is required after the "
                    "decimal point.";
         case yajl_lex_missing_integer_after_minus:
             return "malformed number, a digit is required after the "
                    "minus sign.";
         case yajl_lex_unallowed_comment:
             return "probable comment found in input text, comments are "
                    "not enabled.";
     }
     return "unknown error code";
 }

void yajl_lex_free ( yajl_lexer lexer )

Definition at line 121 of file yajl_lex.c.

 {
     yajl_buf_free(lxr->buf);
     YA_FREE(lxr->alloc, lxr);
     return;
 }

yajl_lex_error yajl_lex_get_error ( yajl_lexer lexer )

allows access to more specific information about the lexical error when yajl_lex_lex returns yajl_tok_error.

Definition at line 733 of file yajl_lex.c.

 {
     if (lexer == NULL) return (yajl_lex_error) -1;
     return lexer->error;
 }

yajl_tok yajl_lex_lex	(	yajl_lexer	lexer,
		const unsigned char *	jsonText,
		size_t	jsonTextLen,
		size_t *	offset,
		const unsigned char **	outBuf,
		size_t *	outLen
	)

run/continue a lex. "offset" is an input/output parameter. It should be initialized to zero for a new chunk of target text, and upon subsetquent calls with the same target text should passed with the value of the previous invocation.

the client may be interested in the value of offset when an error is returned from the lexer. This allows the client to render useful n * error messages.

When you pass the next chunk of data, context should be reinitialized to zero.

Finally, the output buffer is usually just a pointer into the jsonText, however in cases where the entity being lexed spans multiple chunks, the lexer will buffer the entity and the data returned will be a pointer into that buffer.

This behavior is abstracted from client code except for the performance implications which require that the client choose a reasonable chunk size to get adequate performance.

Definition at line 503 of file yajl_lex.c.

 {
     yajl_tok tok = yajl_tok_error;
     unsigned char c;
     size_t startOffset = *offset;
 
     *outBuf = NULL;
     *outLen = 0;
 
     for (;;) {
         assert(*offset <= jsonTextLen);
 
         if (*offset >= jsonTextLen) {
             tok = yajl_tok_eof;
             goto lexed;
         }
 
         c = readChar(lexer, jsonText, offset);
 
         switch (c) {
             case '{':
                 tok = yajl_tok_left_brace;
                 goto lexed;
             case '}':
                 tok = yajl_tok_right_brace;
                 goto lexed;
             case '[':
                 tok = yajl_tok_left_bracket;
                 goto lexed;
             case ']':
                 tok = yajl_tok_right_bracket;
                 goto lexed;
             case ',':
                 tok = yajl_tok_comma;
                 goto lexed;
             case ':':
                 tok = yajl_tok_colon;
                 goto lexed;
             case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
                 startOffset++;
                 break;
             case 't': {
                 const char * want = "rue";
                 do {
                     if (*offset >= jsonTextLen) {
                         tok = yajl_tok_eof;
                         goto lexed;
                     }
                     c = readChar(lexer, jsonText, offset);
                     if (c != *want) {
                         unreadChar(lexer, offset);
                         lexer->error = yajl_lex_invalid_string;
                         tok = yajl_tok_error;
                         goto lexed;
                     }
                 } while (*(++want));
                 tok = yajl_tok_bool;
                 goto lexed;
             }
             case 'f': {
                 const char * want = "alse";
                 do {
                     if (*offset >= jsonTextLen) {
                         tok = yajl_tok_eof;
                         goto lexed;
                     }
                     c = readChar(lexer, jsonText, offset);
                     if (c != *want) {
                         unreadChar(lexer, offset);
                         lexer->error = yajl_lex_invalid_string;
                         tok = yajl_tok_error;
                         goto lexed;
                     }
                 } while (*(++want));
                 tok = yajl_tok_bool;
                 goto lexed;
             }
             case 'n': {
                 const char * want = "ull";
                 do {
                     if (*offset >= jsonTextLen) {
                         tok = yajl_tok_eof;
                         goto lexed;
                     }
                     c = readChar(lexer, jsonText, offset);
                     if (c != *want) {
                         unreadChar(lexer, offset);
                         lexer->error = yajl_lex_invalid_string;
                         tok = yajl_tok_error;
                         goto lexed;
                     }
                 } while (*(++want));
                 tok = yajl_tok_null;
                 goto lexed;
             }
             case '"': {
                 tok = yajl_lex_string(lexer, (const unsigned char *) jsonText,
                                       jsonTextLen, offset);
                 goto lexed;
             }
             case '-':
             case '0': case '1': case '2': case '3': case '4':
             case '5': case '6': case '7': case '8': case '9': {
                 /* integer parsing wants to start from the beginning */
                 unreadChar(lexer, offset);
                 tok = yajl_lex_number(lexer, (const unsigned char *) jsonText,
                                       jsonTextLen, offset);
                 goto lexed;
             }
             case '/':
                 /* hey, look, a probable comment!  If comments are disabled
                  * it's an error. */
                 if (!lexer->allowComments) {
                     unreadChar(lexer, offset);
                     lexer->error = yajl_lex_unallowed_comment;
                     tok = yajl_tok_error;
                     goto lexed;
                 }
                 /* if comments are enabled, then we should try to lex
                  * the thing.  possible outcomes are
                  * - successful lex (tok_comment, which means continue),
                  * - malformed comment opening (slash not followed by
                  *   '*' or '/') (tok_error)
                  * - eof hit. (tok_eof) */
                 tok = yajl_lex_comment(lexer, (const unsigned char *) jsonText,
                                        jsonTextLen, offset);
                 if (tok == yajl_tok_comment) {
                     /* "error" is silly, but that's the initial
                      * state of tok.  guilty until proven innocent. */
                     tok = yajl_tok_error;
                     yajl_buf_clear(lexer->buf);
                     lexer->bufInUse = 0;
                     startOffset = *offset;
                     break;
                 }
                 /* hit error or eof, bail */
                 goto lexed;
             default:
                 lexer->error = yajl_lex_invalid_char;
                 tok = yajl_tok_error;
                 goto lexed;
         }
     }
 
 
   lexed:
     /* need to append to buffer if the buffer is in use or
      * if it's an EOF token */
     if (tok == yajl_tok_eof || lexer->bufInUse) {
         if (!lexer->bufInUse) yajl_buf_clear(lexer->buf);
         lexer->bufInUse = 1;
         yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset);
         lexer->bufOff = 0;
 
         if (tok != yajl_tok_eof) {
             *outBuf = yajl_buf_data(lexer->buf);
             *outLen = yajl_buf_len(lexer->buf);
             lexer->bufInUse = 0;
         }
     } else if (tok != yajl_tok_error) {
         *outBuf = jsonText + startOffset;
         *outLen = *offset - startOffset;
     }
 
     /* special case for strings. skip the quotes. */
     if (tok == yajl_tok_string || tok == yajl_tok_string_with_escapes)
     {
         assert(*outLen >= 2);
         (*outBuf)++;
         *outLen -= 2;
     }
 
 
 #ifdef YAJL_LEXER_DEBUG
     if (tok == yajl_tok_error) {
         printf("lexical error: %s\n",
                yajl_lex_error_to_string(yajl_lex_get_error(lexer)));
     } else if (tok == yajl_tok_eof) {
         printf("EOF hit\n");
     } else {
         printf("lexed %s: '", tokToStr(tok));
         fwrite(*outBuf, 1, *outLen, stdout);
         printf("'\n");
     }
 #endif
 
     return tok;
 }

yajl_tok yajl_lex_peek	(	yajl_lexer	lexer,
		const unsigned char *	jsonText,
		size_t	jsonTextLen,
		size_t	offset
	)

have a peek at the next token, but don't move the lexer forward

Definition at line 749 of file yajl_lex.c.

 {
     const unsigned char * outBuf;
     size_t outLen;
     size_t bufLen = yajl_buf_len(lexer->buf);
     size_t bufOff = lexer->bufOff;
     unsigned int bufInUse = lexer->bufInUse;
     yajl_tok tok;
 
     tok = yajl_lex_lex(lexer, jsonText, jsonTextLen, &offset,
                        &outBuf, &outLen);
 
     lexer->bufOff = bufOff;
     lexer->bufInUse = bufInUse;
     yajl_buf_truncate(lexer->buf, bufLen);
 
     return tok;
 }

Enumerations
enum	yajl_tok { yajl_tok_bool, yajl_tok_colon, yajl_tok_comma, yajl_tok_eof, yajl_tok_error, yajl_tok_left_brace, yajl_tok_left_bracket, yajl_tok_null, yajl_tok_right_brace, yajl_tok_right_bracket, yajl_tok_integer, yajl_tok_double, yajl_tok_string, yajl_tok_string_with_escapes, yajl_tok_comment }

enum	yajl_lex_error { yajl_lex_e_ok = 0, yajl_lex_string_invalid_utf8, yajl_lex_string_invalid_escaped_char, yajl_lex_string_invalid_json_char, yajl_lex_string_invalid_hex_char, yajl_lex_invalid_char, yajl_lex_invalid_string, yajl_lex_missing_integer_after_decimal, yajl_lex_missing_integer_after_exponent, yajl_lex_missing_integer_after_minus, yajl_lex_unallowed_comment }

Typedefs

Enumerations

Functions

Typedef Documentation

Enumeration Type Documentation

Function Documentation