https://github.com/stagas/lexer-next

lexer for recursive descent parsers
https://github.com/stagas/lexer-next

iterable iterator lexer parser parsing recursive-descent-parser regexp regexp-match tokenizer

Last synced: 5 months ago
JSON representation

lexer for recursive descent parsers

• Host: GitHub
• URL: https://github.com/stagas/lexer-next
• Owner: stagas
• License: mit
• Created: 2021-11-03T00:04:29.000Z (over 4 years ago)
• Default Branch: main
• Last Pushed: 2022-12-07T17:28:41.000Z (over 3 years ago)
• Last Synced: 2025-03-10T13:04:36.015Z (over 1 year ago)
• Topics: iterable, iterator, lexer, parser, parsing, recursive-descent-parser, regexp, regexp-match, tokenizer
• Language: TypeScript
• Homepage:
• Size: 1.08 MB
• Stars: 2
• Watchers: 2
• Forks: 0
• Open Issues: 0
• Metadata Files:
  • Readme: README.md
  • License: LICENSE

Awesome Lists containing this project

README

          




lexer-next    





lexer for recursive descent parsers





npm i lexer-next 


pnpm add lexer-next 

yarn add lexer-next

## API

  #  createLexer(tokenize)      src/index.ts#L106  
    
    #  tokenize        


Tokenizer
        
  


createLexer(tokenize)   =>  



#  (input)        
    
    #  input        


string
        
  


(input)   =>  

Lexer





        #  LexerErrorCauses      src/causes.ts#L1  
        
  #  LexerErrorCause      src/causes.ts#L3  
        
  #  constructor(message)      src/causes.ts#L5  
    
  #  new LexerErrorCause()        


LexerErrorCause
      
  #  message        


string
        
  
    
#  message      src/causes.ts#L5  


string
        
#  name   =   'LexerErrorUnknown'    src/causes.ts#L4  


string
        


#  UnexpectedToken      src/causes.ts#L8  
        
  #  constructor(currentToken, expectedGroup, expectedValue)      src/causes.ts#L10  
    
  #  new UnexpectedToken()        


UnexpectedToken
      
  #  currentToken        


Token
        
#  expectedGroup        


string
        
#  expectedValue        


string
        
  
    
#  currentToken      src/causes.ts#L10  


Token
        
#  expectedGroup      src/causes.ts#L10  


string
        
#  expectedValue      src/causes.ts#L10  


string
        
#  message      src/causes.ts#L5  


string
        
#  name   =   'LexerErrorUnexpectedToken'    src/causes.ts#L9  


string
        




#  LexerError      src/index.ts#L15  
        
  #  constructor(cause)      src/index.ts#L21  
    
  #  new LexerError()        


LexerError
      
  #  cause        


Error
        
  
    
#  cause      src/index.ts#L16  


UnexpectedToken
        
#  name   =   'LexerError'    src/index.ts#L20  


string
        
#  prepareStackTrace        
#  (err, stackTraces)        
    
    #  err        


Error
        
#  stackTraces        


CallSite  []
        
  


(err, stackTraces)   =>  

any

    
        #  stackTraceLimit        


number
        
#  captureStackTrace(targetObject, constructorOpt)        
    
    #  targetObject        


object
        
#  constructorOpt        


Function
        
  


captureStackTrace(targetObject, constructorOpt)   =>  

void
    
#  RegExpToken        
        
  #  constructor(pattern)        
    
  #  new RegExpToken()        


RegExpToken
      
  #  pattern        


string | RegExp
        
  
#  new RegExpToken()        


RegExpToken
      
  #  pattern        


string
        
#  flags        


string
        
  
#  new RegExpToken()        


RegExpToken
      
  #  pattern        


string | RegExp
        
#  flags        


string
        
  
    


#  Token        
        
  #  constructor(value)        
    
  #  new Token()        


Token
      
  #  value        


TokenJson
        
  
#  new Token()        


Token
      
  #  value        


string
        
#  group        


string
        
#  source        


RegExpMatchArrayLike
        
  
    
#  group        


string
        
#  source        


RegExpMatchArrayLike
        
#  index        
        
#  value        
        
#  as(value, group)        
    
    #  value        


string
        
#  group        


string
        
  


as(value, group)   =>  

Token

    
#  is(group, value)        
    
    #  group        


string
        
#  value        


string
        
  


is(group, value)   =>  

boolean
    
#  toJSON()        
    
      

toJSON()   =>  

TokenJson
    
#  create(value, group, source)        
    
    #  value        


string
        
#  group        


string
        
#  source        


RegExpMatchArrayLike
        
  


create(value, group, source)   =>  

Token
    
#  Lexer      src/index.ts#L46  
        
  #  unknown      src/index.ts#L89  


Token
        
#  accept(group, value)      src/index.ts#L69  
    
    #  group        


string
        
#  value        


string
        
  


accept(group, value)   =>  



null | Token



    
#  advance()      src/index.ts#L50  
    
      

advance()   =>  

Token
    
#  expect(group, value)      src/index.ts#L77  
    
    #  group        


string
        
#  value        


string
        
  


expect(group, value)   =>  

Token
    
#  filter(fn)      src/index.ts#L87  
    
    #  fn        


FilterFunction
        
  


filter(fn)   =>  

void
    
#  onerror(fn)      src/index.ts#L82  
    
    #  fn        


ErrorHandler
        
  


onerror(fn)   =>  

void
    
#  peek()      src/index.ts#L59  
    
      

peek()   =>  

Token
  #  group        


string
        

#  value        


string
        
  
peek(group, value)   =>  



false | Token

    
#  RegExpMatchArrayLike        
        
  #  index        


number
        
#  input        


string
        


#  TokenJson        
        
  #  group        


string
        
#  source        


{

  #  index        


number
        
#  input        


string
        
#  match        


string  []
        
}
        
#  value        


string
        
#  ErrorHandler      src/index.ts#L31  


#  (error)        
    
    #  error        


LexerError
        
  


(error)   =>  

void

    
        #  FilterFunction      src/index.ts#L39  


#  (token)        
    
    #  token        


Token
        
  


(token)   =>  

boolean

    
        #  LexerFactory      src/index.ts#L95  


#  (input)        
    
    #  input        


string
        
  


(input)   =>  

Lexer

    
        #  Tokenizer      src/index.ts#L41  


#  (input)        
    
    #  input        


string
        
  


(input)   =>  



IterableIterator<RegExpMatchArray>

    
        #  createLexer(tokenize)      src/index.ts#L106  
    
    #  tokenize        


Tokenizer
        
  


createLexer(tokenize)   =>  



#  (input)        
    
    #  input        


string
        
  


(input)   =>  

Lexer
    

    


## Credits

- [match-to-token](https://npmjs.org/package/match-to-token) by [stagas](https://github.com/stagas) – transform a RegExp named group match to a more useful object

## Contributing

[Fork](https://github.com/stagas/lexer-next/fork) or [edit](https://github.dev/stagas/lexer-next) and submit a PR.

All contributions are welcome!

## License

MIT © 2022 [stagas](https://github.com/stagas)