https://github.com/julien-c/tokenizers

https://github.com/julien-c/tokenizers

Last synced: about 16 hours ago
JSON representation

• Host: GitHub
• URL: https://github.com/julien-c/tokenizers
• Owner: julien-c
• License: mit
• Created: 2019-11-27T00:21:56.000Z (almost 5 years ago)
• Default Branch: master
• Last Pushed: 2019-11-27T00:36:53.000Z (almost 5 years ago)
• Last Synced: 2024-06-12T03:01:15.809Z (4 months ago)
• Language: JavaScript
• Size: 301 KB
• Stars: 0
• Watchers: 2
• Forks: 0
• Open Issues: 0
• Metadata Files:
  • Readme: README.md
  • License: LICENSE

Awesome Lists containing this project

README

        
# elasticsearch tokenizers

## Using tokenizers

Install tokenizers from npm

With yarn

    yarn add tokenizers

or alternatively using npm:

    npm install --save tokenizers

___

    const {

      asciiFolding,

      standardTokenizer,

      asciiFoldingTokenizer,

      nGramTokenizerCreater,

      edgeNGramTokenizerCreater,

    } = require('tokenizers');

    asciiFolding('Iлｔèｒｎåｔïｏｎɑｌíƶａｔï߀ԉO')

    // ~ Iлternationɑlizati߀ԉO

    standardTokenizer("Düsseldorf Köln, Москва 北京市 !@#$");

    // ~ [ 'düsseldorf', 'köln', 'москва', '北', '京', '市' ];

    asciiFoldingTokenizer("Düsseldorf Köln, Москва 北京市 !@#$");

    // ~ [ 'dusseldorf', 'koln', 'москва', '北', '京', '市' ];

    const standardNGramTokenizer = nGramTokenizerCreater({

      min: 3,

      max: 4,

      tokenChars: ['letter', 'digit'],

    });

    standardNGramTokenizer('Nam quốc sơn hà nam đế cư');

    // ~ ['nam', 'quố', 'quốc', 'uốc', 'sơn', 'nam']

    const asciifoldingEdgeNGramTokenizer = edgeNGramTokenizerCreater({

      min: 2,

      max: 10,

      filters: ['asciifolding'],

      tokenChars: ['letter', 'digit'],

    });

    asciifoldingEdgeNGramTokenizer('Tiệt nhiên phận định tại thiên thư')

    // ~ ['ti', 'tie', 'tiet', 'nh', 'nhi', 'nhie', 'nhien', 'ph', 'pha', 'phan', 'di', 'din', 'dinh', 'ta', 'tai', 'th', 'thi', 'thie', 'thien', 'th', 'thu']