tokenizer.c File Reference

#include "tokenizer.h"
#include <assert.h>
#include <stdbool.h>
#include <string.h>
#include "attribute.h"
#include "char_ref.h"
#include "error.h"
#include "gumbo.h"
#include "parser.h"
#include "string_buffer.h"
#include "string_piece.h"
#include "token_type.h"
#include "tokenizer_states.h"
#include "utf8.h"
#include "util.h"
#include "vector.h"

Classes
struct	GumboInternalTagState
struct	GumboInternalTokenizerState

Typedefs
typedef struct GumboInternalTagState	GumboTagState
typedef struct GumboInternalTokenizerState	GumboTokenizerState
typedef StateResult(*	GumboLexerStateFunction) (GumboParser *, GumboTokenizerState *, int, GumboToken *)

Enumerations
enum	StateResult { RETURN_ERROR , RETURN_SUCCESS , NEXT_CHAR }

Functions
void	gumbo_tokenizer_state_init (GumboParser *parser, const char *text, size_t text_length)
void	gumbo_tokenizer_state_destroy (GumboParser *parser)
void	gumbo_tokenizer_set_state (GumboParser *parser, GumboTokenizerEnum state)
void	gumbo_tokenizer_set_is_current_node_foreign (GumboParser *parser, bool is_foreign)
bool	gumbo_lex (GumboParser *parser, GumboToken *output)
void	gumbo_token_destroy (GumboParser *parser, GumboToken *token)

Variables
const GumboStringPiece	kScriptTag = {"script", 6}

Typedef Documentation

GumboLexerStateFunction

typedef StateResult(* GumboLexerStateFunction) (GumboParser *, GumboTokenizerState *, int, GumboToken *)

GumboTagState

typedef struct GumboInternalTagState GumboTagState

GumboTokenizerState

typedef struct GumboInternalTokenizerState GumboTokenizerState

Enumeration Type Documentation

StateResult

enum StateResult

Enumerator
RETURN_ERROR
RETURN_SUCCESS
NEXT_CHAR

Function Documentation

gumbo_lex()

bool gumbo_lex	(	GumboParser *	parser,
		GumboToken *	output
	)

gumbo_token_destroy()

void gumbo_token_destroy	(	GumboParser *	parser,
		GumboToken *	token
	)

gumbo_tokenizer_set_is_current_node_foreign()

void gumbo_tokenizer_set_is_current_node_foreign	(	GumboParser *	parser,
		bool	is_foreign
	)

gumbo_tokenizer_set_state()

void gumbo_tokenizer_set_state	(	GumboParser *	parser,
		GumboTokenizerEnum	state
	)

gumbo_tokenizer_state_destroy()

void gumbo_tokenizer_state_destroy

(

GumboParser *

parser

)

gumbo_tokenizer_state_init()

void gumbo_tokenizer_state_init	(	GumboParser *	parser,
		const char *	text,
		size_t	text_length
	)

Variable Documentation

kScriptTag

const GumboStringPiece kScriptTag = {"script", 6}