#include <stdlib.h>
#include <stdio.h>
#include <assert.h>
#include <string.h>
#include "yajl_lex.h"
#include "yajl_buf.h"

Include dependency graph for yajl_lex.c:

Classes
struct	yajl_lexer_t

Macros
#define	readChar(lxr, txt, off)

#define	unreadChar(lxr, off) (((off) > 0) ? ((off))-- : ((lxr)->bufOff--))

#define	VEC 0x01

#define	IJC 0x02

#define	VHC 0x04

#define	NFP 0x08

#define	NUC 0x10

#define	UTF8_CHECK_EOF if (*offset >= jsonTextLen) { return yajl_tok_eof; }

#define	STR_CHECK_EOF

#define	RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof;

Functions
yajl_lexer	yajl_lex_alloc (yajl_alloc_funcs *alloc, unsigned int allowComments, unsigned int validateUTF8)

void	yajl_lex_free (yajl_lexer lxr)

yajl_tok	yajl_lex_lex (yajl_lexer lexer, const unsigned char jsonText, size_t jsonTextLen, size_t offset, const unsigned char *outBuf, size_t outLen)

const char *	yajl_lex_error_to_string (yajl_lex_error error)

yajl_lex_error	yajl_lex_get_error (yajl_lexer lexer)

size_t	yajl_lex_current_line (yajl_lexer lexer)

size_t	yajl_lex_current_char (yajl_lexer lexer)

yajl_tok	yajl_lex_peek (yajl_lexer lexer, const unsigned char *jsonText, size_t jsonTextLen, size_t offset)

Macro Definition Documentation

#define IJC 0x02

Definition at line 137 of file yajl_lex.c.

#define NFP 0x08

Definition at line 139 of file yajl_lex.c.

#define NUC 0x10

Definition at line 140 of file yajl_lex.c.

#define readChar	(	lxr,
		txt,
		off
	)

Value:

(((lxr)->bufInUse && yajl_buf_len((lxr)->buf) && lxr->bufOff < yajl_buf_len((lxr)->buf)) ? \
     (*((const unsigned char *) yajl_buf_data((lxr)->buf) + ((lxr)->bufOff)++)) : \
     ((txt)[(*(off))++]))

Definition at line 96 of file yajl_lex.c.

#define RETURN_IF_EOF if (*offset >= jsonTextLen) return yajl_tok_eof;

Definition at line 372 of file yajl_lex.c.

#define STR_CHECK_EOF

Value:

if (*offset >= jsonTextLen) { \
   tok = yajl_tok_eof; \
   goto finish_string_lex; \
}

Definition at line 248 of file yajl_lex.c.

#define unreadChar	(	lxr,
		off
	)	(((off) > 0) ? ((off))-- : ((lxr)->bufOff--))

Definition at line 101 of file yajl_lex.c.

#define UTF8_CHECK_EOF if (*offset >= jsonTextLen) { return yajl_tok_eof; }

process a variable length utf8 encoded codepoint.

returns: yajl_tok_string - if valid utf8 char was parsed and offset was advanced yajl_tok_eof - if end of input was hit before validation could complete yajl_tok_error - if invalid utf8 was encountered

NOTE: on error the offset will point to the first char of the invalid utf8

Definition at line 196 of file yajl_lex.c.

#define VEC 0x01

Definition at line 136 of file yajl_lex.c.

#define VHC 0x04

Definition at line 138 of file yajl_lex.c.

Function Documentation

yajl_lexer yajl_lex_alloc	(	yajl_alloc_funcs *	alloc,
		unsigned int	allowComments,
		unsigned int	validateUTF8
	)

Definition at line 104 of file yajl_lex.c.

 {
     yajl_lexer lxr = (yajl_lexer) YA_MALLOC(alloc, sizeof(struct yajl_lexer_t));
     if (lxr == NULL) {
         return NULL;
     }
 
     memset((void *) lxr, 0, sizeof(struct yajl_lexer_t));
     lxr->buf = yajl_buf_alloc(alloc);
     lxr->allowComments = allowComments;
     lxr->validateUTF8 = validateUTF8;
     lxr->alloc = alloc;
     return lxr;
 }

size_t yajl_lex_current_char ( yajl_lexer lexer )

get the number of chars lexed by this lexer instance since the last
or

Definition at line 744 of file yajl_lex.c.

 {
     return lexer->charOff;
 }

size_t yajl_lex_current_line ( yajl_lexer lexer )

get the number of lines lexed by this lexer instance

Definition at line 739 of file yajl_lex.c.

 {
     return lexer->lineOff;
 }

const char* yajl_lex_error_to_string ( yajl_lex_error error )

Definition at line 695 of file yajl_lex.c.

 {
     switch (error) {
         case yajl_lex_e_ok:
             return "ok, no error";
         case yajl_lex_string_invalid_utf8:
             return "invalid bytes in UTF8 string.";
         case yajl_lex_string_invalid_escaped_char:
             return "inside a string, '\\' occurs before a character "
                    "which it may not.";
         case yajl_lex_string_invalid_json_char:
             return "invalid character inside string.";
         case yajl_lex_string_invalid_hex_char:
             return "invalid (non-hex) character occurs after '\\u' inside "
                    "string.";
         case yajl_lex_invalid_char:
             return "invalid char in json text.";
         case yajl_lex_invalid_string:
             return "invalid string in json text.";
         case yajl_lex_missing_integer_after_exponent:
             return "malformed number, a digit is required after the exponent.";
         case yajl_lex_missing_integer_after_decimal:
             return "malformed number, a digit is required after the "
                    "decimal point.";
         case yajl_lex_missing_integer_after_minus:
             return "malformed number, a digit is required after the "
                    "minus sign.";
         case yajl_lex_unallowed_comment:
             return "probable comment found in input text, comments are "
                    "not enabled.";
     }
     return "unknown error code";
 }

void yajl_lex_free ( yajl_lexer lxr )

Definition at line 121 of file yajl_lex.c.

 {
     yajl_buf_free(lxr->buf);
     YA_FREE(lxr->alloc, lxr);
     return;
 }

yajl_lex_error yajl_lex_get_error ( yajl_lexer lexer )

allows access to more specific information about the lexical error when yajl_lex_lex returns yajl_tok_error.

Definition at line 733 of file yajl_lex.c.

 {
     if (lexer == NULL) return (yajl_lex_error) -1;
     return lexer->error;
 }

yajl_tok yajl_lex_lex	(	yajl_lexer	lexer,
		const unsigned char *	jsonText,
		size_t	jsonTextLen,
		size_t *	offset,
		const unsigned char **	outBuf,
		size_t *	outLen
	)

run/continue a lex. "offset" is an input/output parameter. It should be initialized to zero for a new chunk of target text, and upon subsetquent calls with the same target text should passed with the value of the previous invocation.

the client may be interested in the value of offset when an error is returned from the lexer. This allows the client to render useful n * error messages.

When you pass the next chunk of data, context should be reinitialized to zero.

Finally, the output buffer is usually just a pointer into the jsonText, however in cases where the entity being lexed spans multiple chunks, the lexer will buffer the entity and the data returned will be a pointer into that buffer.

This behavior is abstracted from client code except for the performance implications which require that the client choose a reasonable chunk size to get adequate performance.

Definition at line 503 of file yajl_lex.c.

 {
     yajl_tok tok = yajl_tok_error;
     unsigned char c;
     size_t startOffset = *offset;
 
     *outBuf = NULL;
     *outLen = 0;
 
     for (;;) {
         assert(*offset <= jsonTextLen);
 
         if (*offset >= jsonTextLen) {
             tok = yajl_tok_eof;
             goto lexed;
         }
 
         c = readChar(lexer, jsonText, offset);
 
         switch (c) {
             case '{':
                 tok = yajl_tok_left_brace;
                 goto lexed;
             case '}':
                 tok = yajl_tok_right_brace;
                 goto lexed;
             case '[':
                 tok = yajl_tok_left_bracket;
                 goto lexed;
             case ']':
                 tok = yajl_tok_right_bracket;
                 goto lexed;
             case ',':
                 tok = yajl_tok_comma;
                 goto lexed;
             case ':':
                 tok = yajl_tok_colon;
                 goto lexed;
             case '\t': case '\n': case '\v': case '\f': case '\r': case ' ':
                 startOffset++;
                 break;
             case 't': {
                 const char * want = "rue";
                 do {
                     if (*offset >= jsonTextLen) {
                         tok = yajl_tok_eof;
                         goto lexed;
                     }
                     c = readChar(lexer, jsonText, offset);
                     if (c != *want) {
                         unreadChar(lexer, offset);
                         lexer->error = yajl_lex_invalid_string;
                         tok = yajl_tok_error;
                         goto lexed;
                     }
                 } while (*(++want));
                 tok = yajl_tok_bool;
                 goto lexed;
             }
             case 'f': {
                 const char * want = "alse";
                 do {
                     if (*offset >= jsonTextLen) {
                         tok = yajl_tok_eof;
                         goto lexed;
                     }
                     c = readChar(lexer, jsonText, offset);
                     if (c != *want) {
                         unreadChar(lexer, offset);
                         lexer->error = yajl_lex_invalid_string;
                         tok = yajl_tok_error;
                         goto lexed;
                     }
                 } while (*(++want));
                 tok = yajl_tok_bool;
                 goto lexed;
             }
             case 'n': {
                 const char * want = "ull";
                 do {
                     if (*offset >= jsonTextLen) {
                         tok = yajl_tok_eof;
                         goto lexed;
                     }
                     c = readChar(lexer, jsonText, offset);
                     if (c != *want) {
                         unreadChar(lexer, offset);
                         lexer->error = yajl_lex_invalid_string;
                         tok = yajl_tok_error;
                         goto lexed;
                     }
                 } while (*(++want));
                 tok = yajl_tok_null;
                 goto lexed;
             }
             case '"': {
                 tok = yajl_lex_string(lexer, (const unsigned char *) jsonText,
                                       jsonTextLen, offset);
                 goto lexed;
             }
             case '-':
             case '0': case '1': case '2': case '3': case '4':
             case '5': case '6': case '7': case '8': case '9': {
                 /* integer parsing wants to start from the beginning */
                 unreadChar(lexer, offset);
                 tok = yajl_lex_number(lexer, (const unsigned char *) jsonText,
                                       jsonTextLen, offset);
                 goto lexed;
             }
             case '/':
                 /* hey, look, a probable comment!  If comments are disabled
                  * it's an error. */
                 if (!lexer->allowComments) {
                     unreadChar(lexer, offset);
                     lexer->error = yajl_lex_unallowed_comment;
                     tok = yajl_tok_error;
                     goto lexed;
                 }
                 /* if comments are enabled, then we should try to lex
                  * the thing.  possible outcomes are
                  * - successful lex (tok_comment, which means continue),
                  * - malformed comment opening (slash not followed by
                  *   '*' or '/') (tok_error)
                  * - eof hit. (tok_eof) */
                 tok = yajl_lex_comment(lexer, (const unsigned char *) jsonText,
                                        jsonTextLen, offset);
                 if (tok == yajl_tok_comment) {
                     /* "error" is silly, but that's the initial
                      * state of tok.  guilty until proven innocent. */
                     tok = yajl_tok_error;
                     yajl_buf_clear(lexer->buf);
                     lexer->bufInUse = 0;
                     startOffset = *offset;
                     break;
                 }
                 /* hit error or eof, bail */
                 goto lexed;
             default:
                 lexer->error = yajl_lex_invalid_char;
                 tok = yajl_tok_error;
                 goto lexed;
         }
     }
 
 
   lexed:
     /* need to append to buffer if the buffer is in use or
      * if it's an EOF token */
     if (tok == yajl_tok_eof || lexer->bufInUse) {
         if (!lexer->bufInUse) yajl_buf_clear(lexer->buf);
         lexer->bufInUse = 1;
         yajl_buf_append(lexer->buf, jsonText + startOffset, *offset - startOffset);
         lexer->bufOff = 0;
 
         if (tok != yajl_tok_eof) {
             *outBuf = yajl_buf_data(lexer->buf);
             *outLen = yajl_buf_len(lexer->buf);
             lexer->bufInUse = 0;
         }
     } else if (tok != yajl_tok_error) {
         *outBuf = jsonText + startOffset;
         *outLen = *offset - startOffset;
     }
 
     /* special case for strings. skip the quotes. */
     if (tok == yajl_tok_string || tok == yajl_tok_string_with_escapes)
     {
         assert(*outLen >= 2);
         (*outBuf)++;
         *outLen -= 2;
     }
 
 
 #ifdef YAJL_LEXER_DEBUG
     if (tok == yajl_tok_error) {
         printf("lexical error: %s\n",
                yajl_lex_error_to_string(yajl_lex_get_error(lexer)));
     } else if (tok == yajl_tok_eof) {
         printf("EOF hit\n");
     } else {
         printf("lexed %s: '", tokToStr(tok));
         fwrite(*outBuf, 1, *outLen, stdout);
         printf("'\n");
     }
 #endif
 
     return tok;
 }

yajl_tok yajl_lex_peek	(	yajl_lexer	lexer,
		const unsigned char *	jsonText,
		size_t	jsonTextLen,
		size_t	offset
	)

have a peek at the next token, but don't move the lexer forward

Definition at line 749 of file yajl_lex.c.

 {
     const unsigned char * outBuf;
     size_t outLen;
     size_t bufLen = yajl_buf_len(lexer->buf);
     size_t bufOff = lexer->bufOff;
     unsigned int bufInUse = lexer->bufInUse;
     yajl_tok tok;
 
     tok = yajl_lex_lex(lexer, jsonText, jsonTextLen, &offset,
                        &outBuf, &outLen);
 
     lexer->bufOff = bufOff;
     lexer->bufInUse = bufInUse;
     yajl_buf_truncate(lexer->buf, bufLen);
 
     return tok;
 }

Classes

Macros

Functions

Macro Definition Documentation

Function Documentation