Ecosyste.ms: Awesome
An open API service indexing awesome lists of open source software.
https://github.com/jamesban/sensitive
敏感词
https://github.com/jamesban/sensitive
gin golang sensitive
Last synced: about 1 month ago
JSON representation
敏感词
- Host: GitHub
- URL: https://github.com/jamesban/sensitive
- Owner: jamesBan
- Created: 2019-05-21T07:20:21.000Z (over 5 years ago)
- Default Branch: master
- Last Pushed: 2021-10-10T13:33:35.000Z (about 3 years ago)
- Last Synced: 2024-10-02T05:23:08.509Z (about 1 month ago)
- Topics: gin, golang, sensitive
- Language: Go
- Size: 4.01 MB
- Stars: 0
- Watchers: 2
- Forks: 0
- Open Issues: 0
-
Metadata Files:
- Readme: README.md
Awesome Lists containing this project
README
# 敏感词过滤服务
## 原理
传统的trie-tree过滤系统容易误判一些容易引起歧义的词语。
利用结巴分词技术，添加敏感词为自定义字库。每次文本分词后检查是否为自定义词组，符合自定义词组的即为敏感词。
## web服务
```go
package main
import (
"time"
"sensitive"
filter2 "sensitive/filter"
store2 "sensitive/store"
"fmt"
"net/http"
"github.com/gin-gonic/gin"
"github.com/gookit/config"
"github.com/gookit/config/yaml"
"os"
"github.com/pkg/errors"
)
type wordForm struct {
Word string `form:"word" binding:"required"`
}
type contentForm struct {
Content string `form:"content" binding:"required"`
}
type replaceForm struct {
Content string `form:"content" binding:"required"`
Replace string `form:"replace" binding:"required"`
}
func init() {
loadConfig()
}
func main() {
gin.SetMode(gin.ReleaseMode)
r := gin.Default()
manager := createManager()
r.POST("/sensitive/web/add", func(c *gin.Context) {
var form wordForm
if c.ShouldBind(&form) != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"message": "missing word param",})
return
}
err := manager.GetStore().Write(form.Word)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
"message": err.Error(),
})
return
}
c.JSON(http.StatusOK, gin.H{
"message": "ok",
})
})
r.DELETE("/sensitive/web/remove", func(c *gin.Context) {
var form wordForm
if c.ShouldBind(&form) != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"message": "missing word param",})
return
}
err := manager.GetStore().Remove(form.Word)
if err != nil {
c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
"message": err.Error(),
})
return
}
c.JSON(http.StatusOK, gin.H{
"message": "ok",
})
})
r.GET("/sensitive/web/check", func(c *gin.Context) {
var form contentForm
if c.BindQuery(&form) != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"message": "missing word param",})
return
}
words := manager.GetFilter().Find(form.Content)
c.AbortWithStatusJSON(http.StatusOK, gin.H{
"words": words,
})
})
r.PUT("/sensitive/web/replace", func(c *gin.Context) {
var form replaceForm
if err := c.ShouldBind(&form); err != nil {
c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"message": err.Error()})
return
}
content := manager.GetFilter().Replace(form.Content, form.Replace)
c.JSON(http.StatusOK, gin.H{
"content": content,
})
})
r.Run(":9512")
}
func loadConfig() {
config.AddDriver(yaml.Driver)
err := config.LoadFiles("config.yaml")
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
}
func createManager() *sensitive.Manager {
store, err := createStore()
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
filter, err := createFilter()
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
interval := config.Int64("interval", 60)
return sensitive.NewManager(store, filter, time.Duration(interval)*time.Second)
}
func createStore() (store2.Store, error) {
driver := config.String("store_dirver")
if driver == "mongo" {
mongoConfig := config.StringMap("mongo_dirver")
dsn := fmt.Sprintf("mongodb://%s:%s@%s:%s/%s", mongoConfig["user"], mongoConfig["password"], mongoConfig["host"], mongoConfig["port"], mongoConfig["name"])
store, err := store2.NewMongoStore(dsn, mongoConfig["name"], mongoConfig["table"], time.Second)
if err != nil {
return nil, errors.Wrapf(err, "connect mongo %s", err)
}
return store, nil
}
return nil, errors.New("error store driver")
}
func createFilter() (filter2.Filter, error) {
filerName := config.String("filter")
if filerName == "jieba" {
pwd, err := os.Getwd()
if err != nil {
return nil, err
}
jiebaConfig := config.StringMap("jieba_filter")
dictPath := jiebaConfig["jieba_filter"]
return filter2.NewJiebaFilter(
pwd+"/"+dictPath+"/jieba.dict.utf8",
pwd+"/"+dictPath+"/hmm_model.utf8",
pwd+"/"+dictPath+"/user.dict.utf8",
pwd+"/"+dictPath+"/idf.utf8",
pwd+"/"+dictPath+"/stop_words.utf8",
), nil
}
return nil, errors.New("error filter name")
}
```
### RPC服务
```go
package main
import (
"time"
"sensitive"
filter2 "sensitive/filter"
store2 "sensitive/store"
"fmt"
)
const (
DB_USER = "root"
DB_PASSWORD = "root"
DB_HOST = "127.0.0.1"
DB_PORET = "27017"
DB_NAME = "test"
DB_COOLECTION = "filter"
DB_INTERVAL = time.Second * 10 //定时更新数据库
)
func main() {
dsn := fmt.Sprintf("mongodb://%s:%s@%s:%s/%s", DB_USER, DB_PASSWORD, DB_HOST, DB_PORET, DB_NAME)
store, err := store2.NewStore(dsn, DB_NAME, DB_COOLECTION, time.Second)
if err != nil {
panic(err)
}
filter := filter2.NewFilter(store)
manager := sensitive.NewManager(store, filter, DB_INTERVAL)
sensitive.RpcServer(manager, "9510")
}
```
参考http://blog.41ms.com/post/40.html