{"id":17988136,"url":"https://github.com/lujun9972/auto-dispatcher.py","last_synced_at":"2025-10-18T08:18:10.683Z","repository":{"id":52375574,"uuid":"65374855","full_name":"lujun9972/auto-dispatcher.py","owner":"lujun9972","description":null,"archived":false,"fork":false,"pushed_at":"2021-04-30T09:24:41.000Z","size":23,"stargazers_count":0,"open_issues_count":0,"forks_count":0,"subscribers_count":3,"default_branch":"master","last_synced_at":"2025-02-09T15:12:30.990Z","etag":null,"topics":[],"latest_commit_sha":null,"homepage":null,"language":"Python","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":null,"status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/lujun9972.png","metadata":{"files":{"readme":"README.org","changelog":null,"contributing":null,"funding":null,"license":null,"code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null}},"created_at":"2016-08-10T10:49:26.000Z","updated_at":"2021-04-30T09:21:45.000Z","dependencies_parsed_at":"2022-08-21T10:00:58.928Z","dependency_job_id":null,"html_url":"https://github.com/lujun9972/auto-dispatcher.py","commit_stats":null,"previous_names":[],"tags_count":0,"template":false,"template_full_name":null,"repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/lujun9972%2Fauto-dispatcher.py","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/lujun9972%2Fauto-dispatcher.py/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/lujun9972%2Fauto-dispatcher.py/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/lujun9972%2Fauto-dispatcher.py/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/lujun9972","download_url":"https://codeload.github.com/lujun9972/auto-dispatcher.py/tar.gz/refs/heads/master","host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":247114421,"owners_count":20885932,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2022-07-04T15:15:14.044Z","host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":[],"created_at":"2024-10-29T19:10:44.712Z","updated_at":"2025-10-18T08:18:05.633Z","avatar_url":"https://github.com/lujun9972.png","language":"Python","funding_links":[],"categories":[],"sub_categories":[],"readme":"#+TITLE: README\n#+AUTHOR: lujun9972\n#+CATEGORY: auto-dispatcher.py\n#+DATE: [2016-07-25 周一 10:07]\n#+OPTIONS: ^:{}\n\n* 功能\n当提交一个安装包到服务器上时,服务器自动根据包名称判断应该使用哪个用户FTP到哪台生产机上,并调用生产机上的升级程序进行升级.\n\n* 设计\n脚本分成两部分,一部分专门检查是否有安装包提交到服务器上,并找出哪些文件是新提交的.\n第二部分根据配置信息上传新提交的文件并执行相应的更像脚本.\n\n*由于生产环境不连外网,因此不能使用第三方库!*\n\n* 实现\n** 判断有新的安装包提交到服务器上\n\n有两种实现方式,一种是通过VC服务器上的Hook实现,该方法要求服务器上安装VC服务器. 另一种方式是通过扫描并对比前后文件树的差别来判断\n\n+ 通过svnlook changed来查询某次提交更改的文件\n  #+BEGIN_SRC sh :tangle \"post-commit\"\n    REPOS_PATH=$1\n    REVISION=$2\n    files=$(svnlook changed ${REPOS_PATH} -r ${REVISION}|egrep -v \"^D\"|egrep -v \"/$\"|cut -f2)\n    auto-dispatcher.py $files\n  #+END_SRC\n+ 扫描并对比前后文件树的差别\n  #+BEGIN_SRC python :tangle \"auto_dispatcher.py\"\n    import sys\n    import os\n    import os.path\n    import time\n    from dispatcher import dispatch_files\n    def directory_files(directory):\n        '''Return a set of full paths of files in DIRECTORY.'''\n        results=[]\n        for root,dirs,files in os.walk(directory):\n            full_paths=[os.path.join(root,f) for f in files]\n            full_paths = [f for f in full_paths if os.path.isfile(f)]\n            results.extend(full_paths)\n        return set(results)\n\n    def directory_files_and_size(directory):\n        '''Return a set of full paths and sizes of files in DIRECTORY.'''\n        files = directory_files(directory)\n        files_and_sizes = set([(f,os.path.getsize(f)) for f in files if os.path.isfile(f)])\n        return files_and_sizes\n\n    def directory_files_until_nochange(directory,interval=5):\n        old = directory_files_and_size(directory)\n        time.sleep(interval)\n        new = directory_files_and_size(directory)\n        while old != new:\n            time.sleep(interval)\n            old,new = new,directory_files_and_size(directory)\n        return set([file_and_size[0] for file_and_size in new])\n\n    if __name__ == \"__main__\":\n        try:\n            directory = sys.argv[1]\n        except IndexError:\n            directory = os.getcwd()\n        old = directory_files(directory)\n        while True:\n            new = directory_files_until_nochange(directory)\n            diff = new - old\n            dispatch_files(diff)\n            time.sleep(60)\n            old = new\n  #+END_SRC\n** 根据配置信息上传新提交的文件并执行相应的更新脚本.\n:PROPERTIES:\n:header-args: :tangle \"dispatcher.py\"\n:END:\n*** 配置日志记录器\n\n初始化日志,日志存放在\"auto-dispatcher.log\"中,最大容量为10M,一共可以有5个备份日志.\n#+BEGIN_SRC conf :tangle \"logger.cfg\"\n  [loggers]\n  keys=root,main\n\n  [logger_root]\n  level=DEBUG\n  handlers=stderr\n\n  [logger_main]\n  level=DEBUG\n  handlers=stderr,file\n  qualname=\"main\"\n  propagate=0\n\n  [handlers]\n  keys=stderr,file\n\n  [handler_stderr]\n  class=StreamHandler\n  formatter=default\n  args=(sys.stderr,)\n\n  [handler_file]\n  class=handlers.RotatingFileHandler\n  formatter=default\n  args=(\"auto-dispatcher.log\",\"a\",10*1024*1024,5)\n\n  [formatters]\n  keys=default\n\n  [formatter_default]\n  format=%(asctime)s [%(thread)d] %(filename)s[line:%(lineno)d] %(levelname)s %(message)s\n#+END_SRC\n\n从logger.cfg中读取logger配置\n#+BEGIN_SRC python \n  # -*- coding: utf-8 -*-\n  import logging\n  import logging.config\n\n  logging.config.fileConfig(\"logger.cfg\")\n  logger = logging.getLogger(\"main\")\n#+END_SRC\n\n\n*** 根据包名找出生产机的用户,地址和登录密码\n需要有一个配置文件(暂时命名为general-dispatch-info.cfg),该配置文件的每一个section的名称都是个正则表达式用于匹配安装包的名称.\n\n配置文件需要以下配置信息\n\n+ 对应生产机IP,必选\n+ 登录生产机的用户,可选,默认从 =~/.netrc= 中获取\n+ 登录生产机的密码,可选,默认从 =~/.netrc= 中获取\n+ 上传到生产机的目录,可选,默认为 =~/newcx/年月日_时分秒/=\n\n下面是配置文件的内容:\n#+BEGIN_SRC conf :tangle \"general-dispatch-info.cfg\"\n  [ibps]\n  host = 10.8.6.10\n  login = ibpsusr\n  password = 123456\n  install_command = touch /tmp/installed\n\n  [cnaps2]\n  host = 10.8.6.10\n  login = cnaps2\n  password = 123456\n#+END_SRC\n\n\n下面定义函数根据包名找出FTP的相关信息\n#+BEGIN_SRC python\n  import netrc\n  import configparser\n  import re\n  import time\n  import sys\n  def get_section_by_package(package,config):\n      '''从config中找出匹配PACKAGE的section. config是ConfigParser.read后的结果'''\n      for section in config.sections():\n          reg = re.compile(section)\n          if reg.match(package):\n              return section\n\n  def get_ftp_info_by_package(package,cfg_file,netrc_file=None):\n      '''根据PACKAGE,从CFG_FILE及NETRC_FILE中找出对应ftp的HOST,LOGIN,ACCOUNT以及PASSWORD\n\n      return host,login,account,password,dest_dir,install_command'''\n      config = configparser.ConfigParser()\n      config.read(cfg_file)\n      section = get_section_by_package(package,config)\n      if not section:\n          logger.warning(\"%s中未找到匹配%s的section\",cfg_file,package)\n          exit(-1)\n      else:\n          try:\n              netrc_info = netrc.netrc(netrc_file)\n              login,account,password = netrc_info.authenticators(host)\n          except Exception:\n              login,account,password = None,None,None\n          host = config.get(section,\"host\")\n          login = config.get(section,\"login\",fallback=None) or login\n          account = config.get(section,\"account\",fallback=None) or account\n          password = config.get(section,\"password\",fallback=None) or password\n          dest_dir = config.get(section,\"dest_dir\",fallback=None) or \"~/newcx/{0}\".format(time.strftime(\"%Y%m%d_%H%M%S\"))\n          install_command = config.get(section,\"install_command\",fallback=None)\n      return host,login,account,password,dest_dir,install_command\n#+END_SRC\n\n#+RESULTS:\n\n*** 登录生产机并在指定目录下上传安装包\n\n**** 若生产机开启ssh服务,则通过scp上传\n\n但是这里遇到两个问题,第一个问题是,执行像ssh,scp这类secure command时,必须手工输入密码,而且它们是直接从控制终端而不是stdin中读取密码的,这也意味着无法通过脚本的方式传送密码給这些程序.\n万幸的是,python中有个名为 =pty= 的modual,它有一个 =spawn= 函数,manual中对它的描述是:\n#+BEGIN_QUOTE\npty.spawn(argv[, master_read[, stdin_read]]) \n    Spawn a process, and connect its controlling terminal with the current process’s standard io. This is often used to baffle programs which insist on reading from the controlling terminal.\n#+END_QUOTE\n这就好办了,我们只要创建一个名为\"pty-process.py\"脚本,在这个脚本中用pty.spawn调用secure command,然后再通过写入该脚本stdin的方式就可以变相地給这些secure command发送密码了.\n\npty-process.py脚本的实现如下:\n#+BEGIN_SRC python :tangle \"pty-process.py\"\n  #!/bin/env python3\n  import pty\n  import sys\n\n  pty.spawn(sys.argv[1:])\n#+END_SRC\n\n借助于这个pty-process.py我们可以实现一个函数用于执行secure command\n#+BEGIN_SRC python\n  def execute_externel_secure_command(command,password=\"\"):\n      secure_command = \"echo {} |python3 pty-process.py {}\".format(password,command)\n      logger.debug(\"execute:%s\",secure_command)\n      result = subprocess.check_output(secure_command,shell=True)\n      logger.debug(\"result:%s\",result)\n      return result\n#+END_SRC\n\n第二个问题是scp并不能自动在远程创建新目录,需要先在远程手工创建目录. 这个解决方案也很简单,直接通过ssh登录远程服务器执行mkdir命令就行:\n#+BEGIN_SRC python\n  def execute_remote_command_by_ssh(host,login,password,command):\n      ssh_command = \"ssh -o StrictHostKeyChecking=no {}@{} '{}'\".format(login,host,command)\n      return execute_externel_secure_command(ssh_command,password)\n#+END_SRC\n\n最后通过scp上传文件的实现为:\n#+BEGIN_SRC python\n  import subprocess\n  def upload_by_scp (file_path,host,dest_dir,login,password):\n      execute_remote_command_by_ssh(host,login,password,\"mkdir -p {}\".format(dest_dir))\n      scp_command = \"scp -o StrictHostKeyChecking=no {0} {1}@{2}:{3}/\".format(file_path,login,host,dest_dir,password)\n      return execute_externel_secure_command(scp_command,password)\n#+END_SRC\n\n**** 若生产机开启FTP服务则通过ftp上传\n#+BEGIN_SRC python\n  import ftplib\n  import os.path\n\n  def upload_by_ftp(file_path,host,dest_dir,login=\"anonymous\",password=\"\",account=\"\"):\n      '''upload FILE_PATH to DEST_DIR in HOST,though ftp protocol'''\n      with ftplib.FTP(host=host,user=login,passwd=password,acct=account) as ftp:\n          ftp.set_debuglevel(2)   # A value of 2 or higher produces the maximum amount of debugging output, logging each line sent and received on the control connection.\n          logger.debug(ftp.getwelcome())\n          try:\n              ftp.mkd(dest_dir)       # 创建目标文件夹\n          except ftplib.error_perm:\n              logger.debug(\"%s:%s already exist\",host,dest_dir)\n          ftp.cwd(dest_dir)       # 进入目标文件夹\n          with open(file_path,\"rb\") as file_handler:\n              ftp.storbinary(\"STOR {0}\".format(os.path.basename(file_path)), file_handler)\n      logger.debug(\"ftp {0} to {1}:{2} done\".format(file_path,host,dest_dir))\n#+END_SRC\n\n\n**** 上传时,优先使用scp上传,若失败则再换成通过ftp上传\n#+BEGIN_SRC python\n  def upload(file_path,host,dest_dir,login,password):\n      '''upload FILE_PATH to DEST_DIR in HOST'''\n      try:\n          upload_by_scp(file_path,host,dest_dir,login,password)\n      except:\n          upload_by_ftp(file_path,host,dest_dir,login,password)\n#+END_SRC\n\n\n*** 调用生产机上的升级程序\n只需要用上面定义的 =execute_remote_command_by_ssh= 就能实现调用生产机上的升级程序了.\n\n*** 分发package\n#+BEGIN_SRC python\n  def dispatch_file(file_path,cfg_file=\"general-dispatch-info.cfg\",netrc_file=None):\n      package = os.path.basename(file_path)\n      host,login,account,password,dest_dir,install_command = get_ftp_info_by_package(package,cfg_file,netrc_file)\n      upload(file_path,host,dest_dir,login,password)\n      if install_command:\n          execute_remote_command_by_ssh(host,login,password,install_command)\n\n  import threading\n  def dispatch_files(file_paths,cfg_file=\"general-dispatch-info.cfg\",netrc_file=None):\n      threads = (threading.Thread(target=dispatch_file,args=(file_path,cfg_file,netrc_file)) for file_path in file_paths)\n      for thread in threads:\n          thread.start()\n      return threads\n#+END_SRC\n\n*** main\n#+BEGIN_SRC python\n  if __name__ == \"__main__\":\n      if len(sys.argv) == 2:\n          dispatch_file(sys.argv[1])\n      else:\n          dispatch_files(sys.argv[1:])\n#+END_SRC\n\n* Local Variables Setting:\n# Local Variables:\n# org-babel-default-header-args:python: ((:session . \"auto_dispatcher\") (:results . \"output\") (:exports . \"code\"))\n# org-babel-python-command: \"python3\"\n# End:\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Flujun9972%2Fauto-dispatcher.py","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Flujun9972%2Fauto-dispatcher.py","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Flujun9972%2Fauto-dispatcher.py/lists"}