{"id":20482149,"url":"https://github.com/uk0/httpwatcher","last_synced_at":"2025-07-13T08:33:43.428Z","repository":{"id":103060184,"uuid":"184085844","full_name":"uk0/httpWatcher","owner":"uk0","description":"http watch on","archived":false,"fork":false,"pushed_at":"2019-05-08T08:12:55.000Z","size":10,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":1,"default_branch":"master","last_synced_at":"2025-07-08T03:16:07.699Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Go","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/uk0.png","metadata":{"files":{"readme":"README.md","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null}},"created_at":"2019-04-29T14:24:11.000Z","updated_at":"2019-05-08T08:12:56.000Z","dependencies_parsed_at":null,"dependency_job_id":"6d1a99c0-93ae-459d-9d7a-4c574b88cb36","html_url":"https://github.com/uk0/httpWatcher","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"purl":"pkg:github/uk0/httpWatcher","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/uk0%2FhttpWatcher","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/uk0%2FhttpWatcher/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/uk0%2FhttpWatcher/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/uk0%2FhttpWatcher/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/uk0","download_url":"https://codeload.github.com/uk0/httpWatcher/tar.gz/refs/heads/master","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/uk0%2FhttpWatcher/sbom","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":265109811,"owners_count":23712752,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-11-15T16:11:47.319Z","updated_at":"2025-07-13T08:33:43.409Z","avatar_url":"https://github.com/uk0.png","language":"Go","funding_links":[],"categories":[],"sub_categories":[],"readme":"# httpWatcher\nhttp watch on\n\n\n```golang\npackage main\n\nimport (\n\t\"bufio\"\n\t\"fmt\"\n\t\"github.com/astaxie/beego/logs\"\n\t\"github.com/nahid/gohttp\"\n\t\"github.com/rs/xid\"\n\t\"io\"\n\t\"os\"\n\t\"strings\"\n\t\"time\"\n)\nvar MiniBatch = 256;\nvar outTime int64 = 9;\nvar start = 0;\n\nvar (\n\tHttpWatcher *WatchClient\n)\nvar WatchThreadNum = 1;\nvar ticker = time.NewTicker(time.Duration(outTime) * time.Second) // --- A\ntype MessageWatcher struct {\n\tstart  int\n\toffset int\n}\n\ntype WatchClient struct {\n\turl string\n\tlineHttpChan chan *MessageWatcher\n}\n\nvar chHttp = make(chan *gohttp.AsyncResponse,100)\n\nfunc NewHttpWatcher( address string) (afk *WatchClient, err error) {\n\tafk = \u0026WatchClient{\n\t\tlineHttpChan:make(chan *MessageWatcher,10),\n\t}\n\tif err != nil {\n\t\tfmt.Printf(\"Failed to create Connetcion: %s\\n\", err)\n\t\tos.Exit(1)\n\t}\n\tafk.url  = address;\n\tfor i:=0;i\u003cWatchThreadNum;i++{\n\t\t// 根据配置文件循环开启线程去发消息到kafka\n\t\tgo PusherKafka()\n\t\tgo afk.Watcher()\n\t}\n\n\treturn\n}\n\n\nfunc InitHttpWatcher()(err error){\n\tvar url = fmt.Sprintf(\"https://2as1.cn/api/sql/query?token=%s\u0026project=%s\", \"21t1rxxxxx\", \"default\")\n\tHttpWatcher,err = NewHttpWatcher(url)\n\treturn\n}\n\n\n\n\nfunc (k *WatchClient) Watcher() {\n\n\t//从channel中读取日志内容放到kafka消息队列中\n\tlogs.Info(\"[start watcher]\")\n\treq := gohttp.NewRequest()\n\tfor v := range k.lineHttpChan {\n\t\tvar formdata =  map[string]string{}\n\t\tformdata[\"q\"] = fmt.Sprintf(\"SELECT * FROM events order by time  limit  %d,%d\", v.start, v.offset)\n\t\tlogs.Info(formdata[\"q\"])\n\n\t\tformdata[\"format\"] = \"json\"\n\t\tvar headerVals  =  map[string]string{}\n\n\t\treq.FormData(formdata).Headers(headerVals).AsyncPost(k.url, chHttp)\n\t}\n}\n\nfunc (k *WatchClient) AddTask(s int, e int) (err error) {\n\tk.lineHttpChan \u003c- \u0026MessageWatcher{start: s, offset: e}\n\treturn\n}\nfunc PusherKafka(){\n\tfor i:=range chHttp{\n\n\t\tif i.Resp.GetResp().StatusCode!=200 || i.Resp.GetResp().ContentLength==0 {\n\t\t\tlogs.Info(\"wait .....\")\n\t\t\tcontinue\n\t\t}else {\n\t\t\tvar result ,_ = i.Resp.GetBodyAsString()\n\n\t\t\trd := bufio.NewReader(strings.NewReader(result))\n\t\t\t// 有数据进行++ 没有数据进行watch on\n\t\t\tstart ++;\n\t\t\tfor {\n\t\t\t\tline, err := rd.ReadString('\\n')\n\t\t\t\tif err != nil || err == io.EOF {\n\t\t\t\t\tbreak\n\t\t\t\t}\n\t\t\t\tline = strings.Replace(line, \"\\f\", \"\", -1)\n\n\t\t\t\t// 执行我们想要的操作\n\t\t\t\tvar newLine = fmt.Sprintf(\",\\\"properties_uuid\\\":\\\"%s\\\"}\",GetUuid())\n\t\t\t\tvar temp = strings.Replace(line,\"}\",newLine,strings.Index(line,\"}\"))\n\t\t\t\tfmt.Printf(\"the line %s\",temp)\n\t\t\t\tHttpSender.addMessage(temp,\"buried-point-mall\")\n\t\t\t}\n\t\t}\n\t}\n}\n\nfunc GetUuid()string{\n\t//h := md5.New()\n\t//h.Write(data)\n\t//s := hex.EncodeToString(h.Sum(nil))\n\t//fmt.Println(s)\n\tguid := xid.New()\n\treturn fmt.Sprintf(\"000000000000%s\",guid.String())\n}\n\n\nfunc main() {\n\tInitHttpWatcher()\n\tInitHttpPusher(true)\n\tloopWorker()\n}\n\nfunc loopWorker(){\n\tdefer ticker.Stop()\n\tfor {\n\t\tselect {\n\t\tcase \u003c-ticker.C:\n\t\t\tHttpWatcher.AddTask( start * MiniBatch,MiniBatch)\n\t\t}\n\t}\n}\n\n```\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fuk0%2Fhttpwatcher","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fuk0%2Fhttpwatcher","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fuk0%2Fhttpwatcher/lists"}