Skip to content

tokens.h#

Tokens for the lexer.

Authors#

TheSilvered

Structs#

Nst_Tok#

Synopsis:

typedef struct _Nst_Tok {
    Nst_Span span;
    Nst_TokType type;
    Nst_ObjRef *value;
} Nst_Tok

Description:

A structure representing a Nest lexer token.

Fields:

  • start: the start position of the token
  • end: the end position of the token
  • type: the type of the token
  • value: the value of the token

Functions#

Nst_tok_new#

Synopsis:

Nst_Tok Nst_tok_new(Nst_Span span, i32 type, Nst_Obj *val)

Description:

Create a new token.

Parameters:

  • span: the span of the token
  • type: the type of the token
  • val: the value of the token, can be NULL

Returns:

The new token.

Nst_tok_invalid#

Synopsis:

Nst_Tok Nst_tok_invalid(void)

Description:

Create a new token with the type set to Nst_TT_INVALID.

Nst_tok_destroy#

Synopsis:

void Nst_tok_destroy(Nst_Tok *token)

Description:

Destroy the contents of a Nst_Tok.

Nst_tok_type_from_str#

Synopsis:

Nst_TokType Nst_tok_type_from_str(u8 *str)

Description:

Get the token type from a string of punctuation characters.

str is expected to be at least one character long.

Parameters:

  • str: the string to parse into the token type

Returns:

The parsed token type or Nst_TT_INVALID if the string does not contain a valid token literal.

Nst_print_tok#

Synopsis:

void Nst_print_tok(Nst_Tok *token)

Description:

Print a token to the Nest standard output.

Nst_tok_type_to_str#

Synopsis:

const char *Nst_tok_type_to_str(Nst_TokType type)

Description:

Convert a Nst_TokType to a string.

Enums#

Nst_TokType#

Synopsis:

typedef enum _Nst_TokType {
    Nst_TT_ADD,       // + + stack-op start, num-op start
    Nst_TT_SUB,       // | |
    Nst_TT_MUL,       // | |
    Nst_TT_DIV,       // | |
    Nst_TT_POW,       // | |
    Nst_TT_MOD,       // | |
    Nst_TT_B_AND,     // | |
    Nst_TT_B_OR,      // | |
    Nst_TT_B_XOR,     // | |
    Nst_TT_LSHIFT,    // | |
    Nst_TT_RSHIFT,    // | - num-op end
    Nst_TT_CONCAT,    // |
    Nst_TT_L_AND,     // | + cond-op start
    Nst_TT_L_OR,      // | - cond-op end
    Nst_TT_L_XOR,     // |
    Nst_TT_GT,        // | + comp-op start
    Nst_TT_LT,        // | |
    Nst_TT_EQ,        // | |
    Nst_TT_NEQ,       // | |
    Nst_TT_GTE,       // | |
    Nst_TT_LTE,       // | - comp-op end
    Nst_TT_CONTAINS,  // - stack-op end
    Nst_TT_CAST,      // + local-stack-op start
    Nst_TT_CALL,      // |
    Nst_TT_SEQ_CALL,  // |
    Nst_TT_THROW,     // |
    Nst_TT_RANGE,     // - local-stack-op end
    Nst_TT_ASSIGN,    // + assignment start
    Nst_TT_ADD_A,     // |
    Nst_TT_SUB_A,     // |
    Nst_TT_MUL_A,     // |
    Nst_TT_DIV_A,     // |
    Nst_TT_POW_A,     // |
    Nst_TT_MOD_A,     // |
    Nst_TT_B_AND_A,   // |
    Nst_TT_B_OR_A,    // |
    Nst_TT_B_XOR_A,   // |
    Nst_TT_LSHIFT_A,  // |
    Nst_TT_RSHIFT_A,  // |
    Nst_TT_CONCAT_A,  // - assignment end
    Nst_TT_LEN,       // + + atom start, local-op start
    Nst_TT_L_NOT,     // | |
    Nst_TT_B_NOT,     // | |
    Nst_TT_STDOUT,    // | |
    Nst_TT_STDIN,     // | |
    Nst_TT_IMPORT,    // | |
    Nst_TT_LOC_CALL,  // | |
    Nst_TT_NEG,       // | |
    Nst_TT_TYPEOF,    // | - local-op end
    Nst_TT_IDENT,     // | + value start
    Nst_TT_VALUE,     // | - value end
    Nst_TT_LAMBDA,    // |
    Nst_TT_L_PAREN,   // |
    Nst_TT_L_BRACE,   // |
    Nst_TT_L_VBRACE,  // - atom end
    Nst_TT_L_BRACKET, // + expr-end start
    Nst_TT_R_PAREN,   // |
    Nst_TT_R_BRACE,   // |
    Nst_TT_R_VBRACE,  // |
    Nst_TT_R_BRACKET, // |
    Nst_TT_IF,        // |
    Nst_TT_AS,        // |
    Nst_TT_ENDL,      // |
    Nst_TT_COMMA,     // |
    Nst_TT_COLON,     // |
    Nst_TT_CATCH,     // |
    Nst_TT_FMT_STR,   // |
    Nst_TT_EOFILE,    // - expr-end end
    Nst_TT_BREAK,     //

    // other tokens

    Nst_TT_EXTRACT,
    Nst_TT_WHILE,
    Nst_TT_DOWHILE,
    Nst_TT_FOR,
    Nst_TT_FUNC,
    Nst_TT_RETURN,
    Nst_TT_SWITCH,
    Nst_TT_CONTINUE,
    Nst_TT_TRY,

    // invalid token

    Nst_TT_INVALID
} Nst_TokType

Description:

Types of tokens generated by the Nest lexer.