import os import json import traceback import threading from concurrent.futures import ThreadPoolExecutor import tornado.ioloop import tornado.web import tornado.concurrent import tornado.gen import libs.log from models.docker import docker from libs.misc import system, mkdirs from iexceptions import PluginNotExistsException, \ PluginSingletonException, \ CommandInvalidateException, \ ContainerLockedException, \ ExecuteException, \ DeleteLockMsgIllegalException from libs.log import app_log from const import START, STOP, OK, FAIL, HTTP_OK, HTTP_MISS_ARGS, HTTP_FAIL, \ HTTP_RESOURCE_NOTFOUNT, RESOURCE_NOTFOUNT, HOST, CONTAINER from settings import PORT lock = threading.RLock() class Runner(object): executor = ThreadPoolExecutor(max_workers=24) def __init__(self): self._plugin_dir = "./plugin" self._running_queue = {} self._isolate = [] self._lock_dir = "./data" @tornado.concurrent.run_on_executor def _async_execute(self, cmd): plugin = cmd.split()[0] cmd = self._plugin_dir + "/" + cmd rc, so, se = system(cmd) if rc: raise ExecuteException(rc=rc, so=so, se=se) return True def require_lock(self, key, lock_msg): return True @staticmethod def read_lock_msg(lock_path): with open(lock_path, 'r') as f: lock_msg = f.read().strip() return lock_msg.strip() def get_lock_msg(self, key): lock_path = self._lock_dir + "/" + key + "/lock" return self.read_lock_msg(lock_path) def delete_lock(self, key, lock_msg): pass @tornado.gen.coroutine def run_cmd(self, cmd, **kwargs): if not self._check_valid(cmd): raise CommandInvalidateException(cmd=cmd) plugin = cmd.split()[0] if not os.path.exists(self._plugin_dir + "/" + plugin): raise PluginNotExistsException(plugin_name=plugin) result = yield self._async_execute(cmd) raise tornado.gen.Return(result) def _check_valid(self, cmd): return True runner = Runner() class PluginHanlder(tornado.web.RequestHandler): def prepare(self): self._runner = runner @tornado.gen.coroutine def run_plugin(self, container_id, plugin_cmd, add_on, plugin_type=CONTAINER): p_array = plugin_cmd.split() plugin = p_array[0] action = p_array[2] operation = p_array[-1] app_log.info("start run plugin:{plugin} cmd:{plugin_cmd}".format( plugin=plugin, plugin_cmd=plugin_cmd)) if plugin_type not in [HOST, CONTAINER]: self.finish({ "code": HTTP_FAIL, "message": "invlidate plugin type", "data": "" }) app_log.info( "invlidate plugin type:{type}".format(type=plugin_type)) return if plugin_type == CONTAINER: if not docker.is_exist(container_id): self.finish({ "code": HTTP_RESOURCE_NOTFOUNT, "message": "can not found container_id in host", "data": "" }) app_log.info("container_id:{container_id} not exists".format( container_id=container_id)) return if operation not in [START, STOP]: self.finish({ "code": HTTP_FAIL, "message": "invlidate operation", "data": "" }) app_log.info( "invlidate operation:{operation}".format(operation=operation)) return lock_key = container_id lock_msg = plugin + ":" + action if plugin_type == HOST: lock_key = "host:" + plugin if operation == START: if self._runner.require_lock(lock_key, lock_msg): try: app_log.info("plugin cmd:{plugin_cmd} " "operation:run plugin status:start" .format(plugin_cmd=plugin_cmd)) result = yield self._runner.run_cmd(plugin_cmd) app_log.info( "plugin cmd:{plugin_cmd} " "operation:run plugin status:success, msg:{result}" .format(plugin_cmd=plugin_cmd, result=result)) self.finish({"code": HTTP_OK, "message": OK, "data": ""}) except Exception as e: app_log.error("plugin cmd:{plugin_cmd} " "operation:none status:failed msg:{msg}" .format(plugin_cmd=plugin_cmd, msg=str(e))) app_log.info("lock_key:{lock_key} " "operation:delete_lock status:start " "msg:due to exception above" .format(lock_key=lock_key)) self._runner.delete_lock(lock_key, lock_msg) app_log.info("lock_key:{lock_key} " "operation:delete_lock status:success" .format(lock_key=lock_key)) self.finish({ "code": HTTP_FAIL, "message": str(e), "data": "" }) else: lock_msg = self._runner.get_lock_msg(lock_key) msg = "there is a job running on lock_key:{lock_key}," \ "log_msg:{log_msg}" \ .format(lock_key=lock_key, lock_msg=lock_msg) self.finish({"code": HTTP_FAIL, "message": msg, "data": ""}) elif operation == STOP: try: app_log.info("plugin cmd:{plugin_cmd} " "operation:run plugin status:start" .format(plugin_cmd=plugin_cmd)) result = yield self._runner.run_cmd(plugin_cmd) self._runner.delete_lock(lock_key, lock_msg) app_log.info( "plugin cmd:{plugin_cmd} " "operation:run plugin status:success, msg:{result}".format( plugin_cmd=plugin_cmd, result=result)) self.finish({ "code": HTTP_OK, "message": OK, "data": "success" }) except IOError as e: app_log.info("lock_key:{lock_key} " "msg:lock file not found" .format(lock_key=lock_key)) self.finish({ "code": HTTP_FAIL, "message": "plugin not running", "data": "" }) except TypeError as e: traceback.print_exc() app_log.info("plugin cmd:{plugin_cmd} msg:type error" .format(plugin_cmd=plugin_cmd)) self.finish({ "code": HTTP_FAIL, "message": "type error", "data": "" }) except Exception as e: app_log.error("plugin cmd:{plugin_cmd} " "operation:none status:failed msg:{msg}".format( plugin_cmd=plugin_cmd, msg=str(e))) self.finish({"code": HTTP_FAIL, "message": str(e), "data": ""}) class DiskHandler(PluginHanlder): @tornado.web.asynchronous def post(self): data = json.loads(self.request.body) try: action = data["action"] container_id = data["container_id"] dirname = data["dirname"] operation = data["operation"] except KeyError as e: self.finish({ "code": HTTP_MISS_ARGS, "message": "miss args %s" % e.args[0] }) rate = "0" if "rate" in data and data["rate"] != "": rate = data["rate"] corrupt_files = data[ "corrupt_files"] if "corrupt_files" in data else "all" cmd = "disk.py -a {action} -d {dirname} -c {container_id} " \ "--corrupt_files {corrupt_files} -r {rate} {operation}" \ .format( action=action, dirname=dirname, container_id=container_id, operation=operation, rate=rate, corrupt_files=corrupt_files) self.run_plugin(container_id, cmd, dirname) class FsHandler(PluginHanlder): @tornado.web.asynchronous def post(self): data = json.loads(self.request.body) try: action = data["action"] methods = data["methods"] errno = data["errno"] random = data["random"] pct = data["pct"] path = data["path"] delay = data["delay"] container_id = data["container_id"] container_ip = data["container_ip"] operation = data["operation"] except KeyError as e: self.finish({"code": HTTP_MISS_ARGS, "message": "miss args %s" % e.args[0]}) cmd = "fs.py -a {action} -m {methods} -e {errno} -r {random} -p {pct} " \ "-P {path} -d {delay} -c {container_ip} {operation}" \ .format( action=action, methods=methods, errno=errno, random=random, pct=pct, path=path, delay=delay, container_ip=container_ip, operation=operation, ) self.run_plugin(container_id, cmd, add_on=None) class NetworkHandler(PluginHanlder): @tornado.web.asynchronous def post(self): data = json.loads(self.request.body) try: action = data["action"] container_id = data["container_id"] container_ip = data["container_ip"] operation = data["operation"] except KeyError as e: self.finish({ "code": HTTP_MISS_ARGS, "message": "miss args %s" % e.args[0] }) rate = "0" if "rate" in data and data["rate"] != "": rate = data["rate"] cmd = "network.py -a {action} --container_id {container_id} " \ "-r {rate} {operation}" \ .format( action=action, container_id=container_id, operation=operation, rate=rate) self.run_plugin(container_id, cmd, container_ip) class CpuHandler(PluginHanlder): @tornado.web.asynchronous def post(self): data = json.loads(self.request.body) try: action = data["action"] container_id = data["container_id"] operation = data["operation"] except KeyError as e: self.finish({ "code": HTTP_MISS_ARGS, "message": "miss args %s" % e.args[0] }) rate = "0" if "rate" in data and data["rate"] != "": rate = data["rate"] cmd = "cpu.py -a {action} --container_id {container_id} " \ "-r {rate} {operation}" \ .format( action=action, container_id=container_id, operation=operation, rate=rate) self.run_plugin(container_id, cmd, "noop") class MemoryHandler(PluginHanlder): @tornado.web.asynchronous def post(self): data = json.loads(self.request.body) try: action = data["action"] container_id = data["container_id"] operation = data["operation"] except KeyError as e: self.finish({ "code": HTTP_MISS_ARGS, "message": "miss args %s" % e.args[0] }) rate = "0" if "rate" in data and data["rate"] != "": rate = data["rate"] cmd = "mem.py -a {action} --container_id {container_id} " \ "-r {rate} {operation}" \ .format( action=action, container_id=container_id, operation=operation, rate=rate) self.run_plugin(container_id, cmd, "noop") class BandwidthHandler(PluginHanlder): @tornado.web.asynchronous def post(self): data = json.loads(self.request.body) try: host_ip = data["host_ip"] operation = data["operation"] action = data["action"] except KeyError as e: self.finish({ "code": HTTP_MISS_ARGS, "message": "miss args %s" % e.args[0] }) egress = "0" if "egress" in data and data["egress"] != "": egress = data["egress"] ingress = "0" if "ingress" in data and data["ingress"] != "": ingress = data["ingress"] latency = "0" if "latency" in data and data["latency"] != "": latency = data["latency"] latency_variation = "0" if "latency_variation" in data and data["latency_variation"] != "": latency_variation = data["latency_variation"] cmd = "bandwidth.py -a {action} --host_ip {host_ip} --egress {egress} " \ "--ingress {ingress} --latency {latency} " \ "--latency_variation {latency_variation} {operation}".format( action=action, host_ip=host_ip, egress=egress, ingress=ingress, latency=latency, latency_variation=latency_variation, operation=operation) self.run_plugin( container_id=None, plugin_cmd=cmd, add_on=None, plugin_type=HOST) class SupportApis(tornado.web.RequestHandler): def get(self): apis = [] disk = { "url": "/api/v1/disk", "description": "disk injection for docker", "args": { "action": ["limit", "full", "fail", "error"], "dirname": "", "container_id": "", "rate": "", "operation": [START, STOP] } } fs = { "url": "/api/v1/fs", "description": "fs injection for docker", "args": { "action": ["delay"], "dirname": "", "container_id": "", "method": "", "latency": "", "operation": [START, STOP] } } network = { "url": "/api/v1/network", "description": "network injection for docker", "args": { "action": ["delay", "fail", "loss"], "container_id": "", "container_ip": "", "rate": "", "operation": [START, STOP] } } mem = { "url": "/api/v1/memory", "description": "memory injection for docker", "args": { "action": ["limit"], "container_id": "", "rate": "", "operation": [START, STOP] } } cpu = { "url": "/api/v1/cpu", "description": "cpu injection for docker", "args": { "action": ["limit"], "container_id": "", "rate": "", "operation": [START, STOP] } } bandwidth = { "url": "api/v1/bandwidth", "description": "set network bandwidth for host", "args": { "action": "limit", "host_ip": "", "egress": "", "ingress": "", "latency": "", "latency_variation": "", "operation": [START, STOP] } } apis.append(disk) apis.append(fs) apis.append(network) apis.append(mem) apis.append(cpu) apis.append(bandwidth) ret = {"code": 200, "message": "OK", "data": apis} self.finish(json.dumps(ret)) def make_app(): return tornado.web.Application([ (r"/api/v1/disk", DiskHandler), (r"/api/v1/fs", FsHandler), (r"/api/v1/network", NetworkHandler), (r"/api/v1/bandwidth", BandwidthHandler), (r"/api/v1/cpu", CpuHandler), (r"/api/v1/memory", MemoryHandler), (r"/api/v1/supportapis", SupportApis) ], ) if __name__ == "__main__": app = make_app() app.listen(PORT) tornado.ioloop.IOLoop.current().start()