diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..6e51dee2 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.vue linguist-language=python diff --git a/.github/ISSUE_TEMPLATE/1bug.yaml b/.github/ISSUE_TEMPLATE/1bug.yaml new file mode 100644 index 00000000..b7e98c65 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/1bug.yaml @@ -0,0 +1,60 @@ +name: Bug Report +description: File a bug report +title: "[Bug]: " +labels: ["☢️ bug"] +assignees: + - Selina316 +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: dropdown + id: aspects + attributes: + label: This bug is related to UI or API? + multiple: true + options: + - UI + - API + - type: textarea + id: happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + value: "A bug happened!" + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: What version of our software are you running? + value: "newest" + validations: + required: true + - type: dropdown + id: browsers + attributes: + label: What browsers are you seeing the problem on? + multiple: true + options: + - Firefox + - Chrome + - Safari + - Microsoft Edge + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell diff --git a/.github/ISSUE_TEMPLATE/2feature.yaml b/.github/ISSUE_TEMPLATE/2feature.yaml new file mode 100644 index 00000000..e5ce3230 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/2feature.yaml @@ -0,0 +1,44 @@ +name: Feature wanted +description: A new feature would be good +title: "[Feature]: " +labels: ["✏️ feature"] +assignees: + - pycook +body: + - type: markdown + attributes: + value: | + Thank you for your feature suggestion; we will evaluate it carefully! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: dropdown + id: aspects + attributes: + label: feature is related to UI or API aspects? + multiple: true + options: + - UI + - API + - type: textarea + id: feature + attributes: + label: What is your advice? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you want! + value: "everyone wants this feature!" + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: What version of our software are you running? + value: "newest" + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/3consultation.yaml b/.github/ISSUE_TEMPLATE/3consultation.yaml new file mode 100644 index 00000000..1e85ad64 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/3consultation.yaml @@ -0,0 +1,36 @@ +name: Help wanted +description: I have a question +title: "[help wanted]: " +labels: ["help wanted"] +assignees: + - ivonGwy +body: + - type: markdown + attributes: + value: | + Please tell us what's you need! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: textarea + id: question + attributes: + label: What is your question? + description: Also tell us, how can we help? + placeholder: Tell us what you need! + value: "i have a question!" + validations: + required: true + - type: input + id: version + attributes: + label: Version + description: What version of our software are you running? + value: "newest" + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml new file mode 100644 index 00000000..c21d7d89 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yaml @@ -0,0 +1,60 @@ +name: Bug Report +description: File a bug report +title: "[Bug]: " +labels: ["bug"] +assignees: + - pycook +body: + - type: markdown + attributes: + value: | + Thanks for taking the time to fill out this bug report! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: dropdown + id: type + attributes: + label: bug is related to UI or API aspects? + multiple: true + options: + - UI + - API + - type: textarea + id: what-happened + attributes: + label: What happened? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you see! + value: "A bug happened!" + validations: + required: true + - type: textarea + id: version + attributes: + label: Version + description: What version of our software are you running? + default: 2.3.5 + validations: + required: true + - type: dropdown + id: browsers + attributes: + label: What browsers are you seeing the problem on? + multiple: true + options: + - Firefox + - Chrome + - Safari + - Microsoft Edge + - type: textarea + id: logs + attributes: + label: Relevant log output + description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. + render: shell diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..76847109 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,6 @@ +blank_issues_enabled: false +contact_links: + - name: veops official website + url: https://veops.cn/#hero + about: you can contact us here. + diff --git a/cmdb-api/README.md b/.github/ISSUE_TEMPLATE/consultation.yaml similarity index 100% rename from cmdb-api/README.md rename to .github/ISSUE_TEMPLATE/consultation.yaml diff --git a/.github/ISSUE_TEMPLATE/feature.yaml b/.github/ISSUE_TEMPLATE/feature.yaml new file mode 100644 index 00000000..4e830583 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yaml @@ -0,0 +1,44 @@ +name: Feature wanted +description: A new feature would be good +title: "[Feature]: " +labels: ["feature"] +assignees: + - pycook +body: + - type: markdown + attributes: + value: | + Thank you for your feature suggestion; we will evaluate it carefully! + - type: input + id: contact + attributes: + label: Contact Details + description: How can we get in touch with you if we need more info? + placeholder: ex. email@example.com + validations: + required: false + - type: dropdown + id: type + attributes: + label: feature is related to UI or API aspects? + multiple: true + options: + - UI + - API + - type: textarea + id: describe the feature + attributes: + label: What is your advice? + description: Also tell us, what did you expect to happen? + placeholder: Tell us what you want! + value: "everyone wants this feature!" + validations: + required: true + - type: textarea + id: version + attributes: + label: Version + description: What version of our software are you running? + default: 2.3.5 + validations: + required: true diff --git a/.github/config.yml b/.github/config.yml new file mode 100644 index 00000000..e69de29b diff --git a/.gitignore b/.gitignore index a53b01e6..6c63d758 100755 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,8 @@ pip-log.txt nosetests.xml .pytest_cache cmdb-api/test-output +cmdb-api/api/uploaded_files +cmdb-api/migrations/versions # Translations *.mo @@ -68,6 +70,7 @@ settings.py # UI cmdb-ui/node_modules cmdb-ui/dist +cmdb-ui/yarn.lock # Log files cmdb-ui/npm-debug.log* diff --git a/Makefile b/Makefile index 99b1122e..8a597427 100644 --- a/Makefile +++ b/Makefile @@ -1,37 +1,52 @@ -.PHONY: env clean api ui worker - -help: - @echo " env create a development environment using pipenv" - @echo " deps install dependencies using pip" - @echo " clean remove unwanted files like .pyc's" - @echo " lint check style with flake8" - @echo " api start api server" - @echo " ui start ui server" - @echo " worker start async tasks worker" - -env: +MYSQL_ROOT_PASSWORD ?= root +MYSQL_PORT ?= 3306 +REDIS_PORT ?= 6379 + +default: help +help: ## display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) +.PHONY: help + +env: ## create a development environment using pipenv sudo easy_install pip && \ - pip install pipenv -i https://pypi.douban.com/simple && \ + pip install pipenv -i https://repo.huaweicloud.com/repository/pypi/simple && \ npm install yarn && \ make deps +.PHONY: env + +docker-mysql: ## deploy MySQL use docker + @docker run --name mysql -p ${MYSQL_PORT}:3306 -e MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} -d mysql:latest +.PHONY: docker-mysql + +docker-redis: ## deploy Redis use docker + @docker run --name redis -p ${REDIS_PORT}:6379 -d redis:latest +.PHONY: docker-redis -deps: +deps: ## install dependencies using pip + cd cmdb-api && \ pipenv install --dev && \ pipenv run flask db-setup && \ pipenv run flask cmdb-init-cache && \ + cd .. && \ cd cmdb-ui && yarn install && cd .. +.PHONY: deps -api: +api: ## start api server cd cmdb-api && pipenv run flask run -h 0.0.0.0 +.PHONY: api -worker: - cd cmdb-api && pipenv run celery worker -A celery_worker.celery -E -Q one_cmdb_async --concurrency=1 -D && pipenv run celery worker -A celery_worker.celery -E -Q acl_async --concurrency=1 -D +worker: ## start async tasks worker + cd cmdb-api && pipenv run celery -A celery_worker.celery worker -E -Q one_cmdb_async --autoscale=5,2 --logfile=one_cmdb_async.log -D && pipenv run celery -A celery_worker.celery worker -E -Q acl_async --autoscale=2,1 --logfile=one_acl_async.log -D +.PHONY: worker -ui: +ui: ## start ui server cd cmdb-ui && yarn run serve +.PHONY: ui -clean: +clean: ## remove unwanted files like .pyc's pipenv run flask clean +.PHONY: clean -lint: +lint: ## check style with flake8 flake8 --exclude=env . +.PHONY: lint diff --git a/README.md b/README.md index d85e02bb..e752805f 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,21 @@ -![基础资源视图](docs/logo.png) -[![License](https://img.shields.io/badge/License-AGPLv3-brightgreen)](https://github.com/veops/cmdb/blob/master/LICENSE) -[![UI](https://img.shields.io/badge/UI-Ant%20Design%20Pro%20Vue-brightgreen)](https://github.com/sendya/ant-design-pro-vue) -[![API](https://img.shields.io/badge/API-Flask-brightgreen)](https://github.com/pallets/flask) +

+ 维易CMDB +

+

简单、轻量、通用的运维配置管理数据库

+

+ License: GPLv3 + UI + API +

-[English](README_en.md) / [中文](README.md) -- 在线体验: CMDB - - username: demo +------------------------------ + +[English](docs/README_en.md) / [中文](README.md) +- 产品文档:https://veops.cn/docs/ +- 在线体验:CMDB + - username: demo 或者 admin - password: 123456 > **重要提示**: `master` 分支在开发过程中可能处于 _不稳定的状态_ 。 @@ -15,45 +23,42 @@ ## 系统介绍 -### 整体架构 +### 系统概览 - + -### 相关文档 +[查看更多展示](docs/screenshot.md) -- 设计文档 +### 相关文章 + +- 概要设计 - API 文档 -- 树形视图实践 +- 自动发现 +- 更多文章可以在公众号 **维易科技OneOps** 里查看 ### 特点 - 灵活性 - 1. 规范并统一纳管复杂数据资产 - 2. 自动发现、入库 IT 资产 + 1. 配置灵活,不设定任何运维场景,有内置模板 + 2. 自动发现、入库 IT 资产 - 安全性 - 1. 细粒度访问控制 + 1. 细粒度权限控制 2. 完备操作日志 - 多应用 1. 丰富视图展示维度 - 2. 提供 Restful API - 3. 自定义字段触发器 + 2. API简单强大 + 3. 支持定义属性触发器、计算属性 ### 主要功能 - 模型属性支持索引、多值、默认排序、字体颜色,支持计算属性 - 支持自动发现、定时巡检、文件导入 -- 支持资源、树形、关系视图展示 +- 支持资源、层级、关系视图展示 - 支持模型间关系配置和展示 - 细粒度访问控制,完备的操作日志 - 支持跨模型搜索 -### 系统概览 - -- 服务树 - -![1](docs/0.png "首页展示") -[查看更多展示](docs/screenshot.md) @@ -63,18 +68,27 @@ ## 接入公司 -> 欢迎使用CMDB的公司,在 [#112](https://github.com/veops/cmdb/issues/112) 登记 +> 欢迎使用开源CMDB的公司,在 [#112](https://github.com/veops/cmdb/issues/112) 登记 ## 安装 -### [Docker 一键快速构建](docs/docker.md) +### Docker 一键快速构建 +- 进入主目录(先安装 docker 环境, 注意要clone整个项目) + +``` +docker-compose up -d +``` + +- 浏览器打开: [http://127.0.0.1:8000](http://127.0.0.1:8000) +- username: demo 或者 admin +- password: 123456 -### [本地搭建](docs/local.md) +### [本地开发环境搭建](docs/local.md) ### [Makefile 安装](docs/makefile.md) --- -_**欢迎关注我们的公众号,点击联系我们,加入微信、qq运维群,获得更多产品、行业相关资讯**_ +_**欢迎关注公众号(维易科技OneOps),关注后可加入微信群,进行产品和技术交流。**_ -![公众号](docs/qrcode_for_gzh.jpg) +![公众号: 维易科技OneOps](docs/images/wechat.png) diff --git a/cmdb-api/Makefile b/cmdb-api/Makefile deleted file mode 100644 index a289b86d..00000000 --- a/cmdb-api/Makefile +++ /dev/null @@ -1,16 +0,0 @@ -default: help - -test: ## test in local environment - pytest -s --html=test-output/test/index.html --cov-report html:test-output/coverage --cov=api tests - -clean_test: ## clean test output - rm -f .coverage - rm -rf .pytest_cache - rm -rf test-output - - -docker_test: ## test all case in docker container - @echo "TODO" - -help: - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' ./Makefile | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' diff --git a/cmdb-api/Pipfile b/cmdb-api/Pipfile index 3402e75f..4ae82247 100644 --- a/cmdb-api/Pipfile +++ b/cmdb-api/Pipfile @@ -5,60 +5,63 @@ name = "pypi" [packages] # Flask -Flask = "==1.0.3" -Werkzeug = "==0.15.5" +Flask = "==2.3.2" +Werkzeug = ">=2.3.6" click = ">=5.0" # Api -Flask-RESTful = "==0.3.7" +Flask-RESTful = "==0.3.10" # Database -Flask-SQLAlchemy = "==2.4.0" -SQLAlchemy = "==1.3.5" -PyMySQL = "==0.9.3" -redis = "==3.2.1" +Flask-SQLAlchemy = "==2.5.0" +SQLAlchemy = "==1.4.49" +PyMySQL = "==1.1.0" +redis = "==4.6.0" # Migrations Flask-Migrate = "==2.5.2" # Deployment -gunicorn = "==19.5.0" +gunicorn = "==21.0.1" supervisor = "==4.0.3" # Auth -Flask-Login = "==0.4.1" -Flask-Bcrypt = "==0.7.1" +Flask-Login = ">=0.6.2" +Flask-Bcrypt = "==1.0.1" Flask-Cors = ">=3.0.8" -python-ldap = "==3.2.0" +ldap3 = "==2.9.1" pycryptodome = "==3.12.0" +cryptography = ">=41.0.2" # Caching Flask-Caching = ">=1.0.0" # Environment variable parsing environs = "==4.2.0" marshmallow = "==2.20.2" # async tasks -celery = "==4.3.0" +celery = ">=5.3.1" celery_once = "==3.0.1" more-itertools = "==5.0.0" -kombu = "==4.4.0" +kombu = ">=5.3.1" # common setting -Flask-APScheduler = "==1.12.4" timeout-decorator = "==0.5.0" -numpy = "==1.18.5" -pandas = "==1.3.2" WTForms = "==3.0.0" email-validator = "==1.3.1" treelib = "==1.6.1" flasgger = "==0.9.5" -Pillow = "==8.3.2" +Pillow = ">=10.0.1" # other -six = "==1.12.0" +six = "==1.16.0" bs4 = ">=0.0.1" toposort = ">=1.5" requests = ">=2.22.0" +requests_oauthlib = "==1.3.1" +markdownify = "==0.11.6" PyJWT = "==2.4.0" elasticsearch = "==7.17.9" -future = "==0.18.2" -itsdangerous = "==2.0.1" -Jinja2 = "==3.0.1" +future = "==0.18.3" +itsdangerous = "==2.1.2" +Jinja2 = "==3.1.2" jinja2schema = "==0.1.4" msgpack-python = "==0.5.6" alembic = "==1.7.7" +hvac = "==2.0.0" +colorama = ">=0.4.6" +pycryptodomex = ">=3.19.0" [dev-packages] # Testing @@ -75,4 +78,3 @@ flake8-isort = "==2.7.0" isort = "==4.3.21" pep8-naming = "==0.8.2" pydocstyle = "==3.0.0" - diff --git a/cmdb-api/api/app.py b/cmdb-api/api/app.py index be828fe8..6ea299d3 100644 --- a/cmdb-api/api/app.py +++ b/cmdb-api/api/app.py @@ -7,31 +7,25 @@ import sys from inspect import getmembers from logging.handlers import RotatingFileHandler +from pathlib import Path from flask import Flask -from flask import make_response, jsonify +from flask import jsonify +from flask import make_response from flask.blueprints import Blueprint from flask.cli import click -from flask.json import JSONEncoder +from flask.json.provider import DefaultJSONProvider import api.views.entry -from api.extensions import ( - bcrypt, - cors, - cache, - db, - login_manager, - migrate, - celery, - rd, - es, -) +from api.extensions import (bcrypt, cache, celery, cors, db, es, login_manager, migrate, rd) +from api.extensions import inner_secrets from api.flask_cas import CAS +from api.lib.secrets.secrets import InnerKVManger from api.models.acl import User HERE = os.path.abspath(os.path.dirname(__file__)) PROJECT_ROOT = os.path.join(HERE, os.pardir) -API_PACKAGE = "api" +BASE_DIR = Path(__file__).resolve().parent.parent @login_manager.user_loader @@ -75,7 +69,7 @@ def __call__(self, environ, start_response): return self.app(environ, start_response) -class MyJSONEncoder(JSONEncoder): +class MyJSONEncoder(DefaultJSONProvider): def default(self, o): if isinstance(o, (decimal.Decimal, datetime.date, datetime.time)): return str(o) @@ -86,15 +80,6 @@ def default(self, o): return o -def create_acl_app(config_object="settings"): - app = Flask(__name__.split(".")[0]) - app.config.from_object(config_object) - - register_extensions(app) - - return app - - def create_app(config_object="settings"): """Create application factory, as explained here: http://flask.pocoo.org/docs/patterns/appfactories/. @@ -103,7 +88,7 @@ def create_app(config_object="settings"): app = Flask(__name__.split(".")[0]) app.config.from_object(config_object) - app.json_encoder = MyJSONEncoder + app.json = MyJSONEncoder(app) configure_logger(app) register_extensions(app) register_blueprints(app) @@ -135,12 +120,18 @@ def register_extensions(app): db.init_app(app) cors.init_app(app) login_manager.init_app(app) - migrate.init_app(app, db) + migrate.init_app(app, db, directory=f"{BASE_DIR}/migrations") rd.init_app(app) if app.config.get('USE_ES'): es.init_app(app) + + app.config.update(app.config.get("CELERY")) celery.conf.update(app.config) + if app.config.get('SECRETS_ENGINE') == 'inner': + with app.app_context(): + inner_secrets.init_app(app, InnerKVManger()) + def register_blueprints(app): for item in getmembers(api.views.entry): @@ -158,10 +149,8 @@ def render_error(error): error_code = getattr(error, "code", 500) if not str(error_code).isdigit(): error_code = 400 - if error_code != 500: - return make_response(jsonify(message=str(error)), error_code) - else: - return make_response(jsonify(message=traceback.format_exc(-1)), error_code) + + return make_response(jsonify(message=str(error)), error_code) for errcode in app.config.get("ERROR_CODES") or [400, 401, 403, 404, 405, 500, 502]: app.errorhandler(errcode)(render_error) @@ -184,9 +173,8 @@ def register_commands(app): for root, _, files in os.walk(os.path.join(HERE, "commands")): for filename in files: if not filename.startswith("_") and filename.endswith("py"): - module_path = os.path.join(API_PACKAGE, root[root.index("commands"):]) - if module_path not in sys.path: - sys.path.insert(1, module_path) + if root not in sys.path: + sys.path.insert(1, root) command = __import__(os.path.splitext(filename)[0]) func_list = [o[0] for o in getmembers(command) if isinstance(o[1], click.core.Command)] for func_name in func_list: diff --git a/cmdb-api/api/commands/click_acl.py b/cmdb-api/api/commands/click_acl.py index d35c6a36..3588d8c5 100644 --- a/cmdb-api/api/commands/click_acl.py +++ b/cmdb-api/api/commands/click_acl.py @@ -1,10 +1,15 @@ import click from flask.cli import with_appcontext +from api.lib.perm.acl.user import UserCRUD + @click.command() @with_appcontext def init_acl(): + """ + acl init + """ from api.models.acl import Role from api.models.acl import App from api.tasks.acl import role_rebuild @@ -20,50 +25,18 @@ def init_acl(): role_rebuild.apply_async(args=(role.id, app.id), queue=ACL_QUEUE) -# @click.command() -# @with_appcontext -# def acl_clean(): -# from api.models.acl import Resource -# from api.models.acl import Permission -# from api.models.acl import RolePermission -# -# perms = RolePermission.get_by(to_dict=False) -# -# for r in perms: -# perm = Permission.get_by_id(r.perm_id) -# if perm and perm.app_id != r.app_id: -# resource_id = r.resource_id -# resource = Resource.get_by_id(resource_id) -# perm_name = perm.name -# existed = Permission.get_by(resource_type_id=resource.resource_type_id, name=perm_name, first=True, -# to_dict=False) -# if existed is not None: -# other = RolePermission.get_by(rid=r.rid, perm_id=existed.id, resource_id=resource_id) -# if not other: -# r.update(perm_id=existed.id) -# else: -# r.soft_delete() -# else: -# r.soft_delete() -# -# -# @click.command() -# @with_appcontext -# def acl_has_resource_role(): -# from api.models.acl import Role -# from api.models.acl import App -# from api.lib.perm.acl.cache import HasResourceRoleCache -# from api.lib.perm.acl.role import RoleCRUD -# -# roles = Role.get_by(to_dict=False) -# apps = App.get_by(to_dict=False) -# for role in roles: -# if role.app_id: -# res = RoleCRUD.recursive_resources(role.id, role.app_id) -# if res.get('resources') or res.get('groups'): -# HasResourceRoleCache.add(role.id, role.app_id) -# else: -# for app in apps: -# res = RoleCRUD.recursive_resources(role.id, app.id) -# if res.get('resources') or res.get('groups'): -# HasResourceRoleCache.add(role.id, app.id) +@click.command() +@with_appcontext +def add_user(): + """ + create a user + + is_admin: default is False + + """ + + username = click.prompt('Enter username', confirmation_prompt=False) + password = click.prompt('Enter password', hide_input=True, confirmation_prompt=True) + email = click.prompt('Enter email ', confirmation_prompt=False) + + UserCRUD.add(username=username, password=password, email=email) diff --git a/cmdb-api/api/commands/click_cmdb.py b/cmdb-api/api/commands/click_cmdb.py index 735464d8..0e428045 100644 --- a/cmdb-api/api/commands/click_cmdb.py +++ b/cmdb-api/api/commands/click_cmdb.py @@ -7,13 +7,15 @@ import time import click +import requests from flask import current_app from flask.cli import with_appcontext +from flask_login import login_user import api.lib.cmdb.ci from api.extensions import db from api.extensions import rd -from api.lib.cmdb.ci_type import CITypeTriggerManager +from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import REDIS_PREFIX_CI from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION @@ -22,13 +24,17 @@ from api.lib.cmdb.const import ValueTypeEnum from api.lib.exception import AbortException from api.lib.perm.acl.acl import ACLManager +from api.lib.perm.acl.acl import UserCache from api.lib.perm.acl.cache import AppCache from api.lib.perm.acl.resource import ResourceCRUD from api.lib.perm.acl.resource import ResourceTypeCRUD from api.lib.perm.acl.role import RoleCRUD -from api.lib.perm.acl.user import UserCRUD +from api.lib.secrets.inner import KeyManage +from api.lib.secrets.inner import global_key_threshold +from api.lib.secrets.secrets import InnerKVManger from api.models.acl import App from api.models.acl import ResourceType +from api.models.cmdb import Attribute from api.models.cmdb import CI from api.models.cmdb import CIRelation from api.models.cmdb import CIType @@ -50,6 +56,7 @@ def cmdb_init_cache(): if relations: rd.create_or_update(relations, REDIS_PREFIX_CI_RELATION) + es = None if current_app.config.get("USE_ES"): from api.extensions import es from api.models.cmdb import Attribute @@ -120,10 +127,10 @@ def cmdb_init_acl(): # 3. add resource and grant ci_types = CIType.get_by(to_dict=False) - type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id + resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id for ci_type in ci_types: try: - ResourceCRUD.add(ci_type.name, type_id, app_id) + ResourceCRUD.add(ci_type.name, resource_type_id, app_id) except AbortException: pass @@ -133,10 +140,10 @@ def cmdb_init_acl(): [PermEnum.READ]) relation_views = PreferenceRelationView.get_by(to_dict=False) - type_id = ResourceType.get_by(name=ResourceTypeEnum.RELATION_VIEW, first=True, to_dict=False).id + resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.RELATION_VIEW, first=True, to_dict=False).id for view in relation_views: try: - ResourceCRUD.add(view.name, type_id, app_id) + ResourceCRUD.add(view.name, resource_type_id, app_id) except AbortException: pass @@ -147,61 +154,15 @@ def cmdb_init_acl(): @click.command() -@click.option( - '-u', - '--user', - help='username' -) -@click.option( - '-p', - '--password', - help='password' -) -@click.option( - '-m', - '--mail', - help='mail' -) @with_appcontext -def add_user(user, password, mail): - """ - create a user - - is_admin: default is False - - Example: flask add-user -u -p -m - """ - assert user is not None - assert password is not None - assert mail is not None - UserCRUD.add(username=user, password=password, email=mail) - - -@click.command() -@click.option( - '-u', - '--user', - help='username' -) -@with_appcontext -def del_user(user): +def cmdb_counter(): """ - delete a user - - Example: flask del-user -u + Dashboard calculations """ - assert user is not None - from api.models.acl import User - - u = User.get_by(username=user, first=True, to_dict=False) - u and UserCRUD.delete(u.uid) - - -@click.command() -@with_appcontext -def cmdb_counter(): from api.lib.cmdb.cache import CMDBCounterCache + current_app.test_request_context().push() + login_user(UserCache.get('worker')) while True: try: db.session.remove() @@ -217,45 +178,283 @@ def cmdb_counter(): @click.command() @with_appcontext def cmdb_trigger(): + """ + Trigger execution for date attribute + """ + from api.lib.cmdb.ci import CITriggerManager + current_day = datetime.datetime.today().strftime("%Y-%m-%d") trigger2cis = dict() trigger2completed = dict() i = 0 while True: - db.session.remove() - if datetime.datetime.today().strftime("%Y-%m-%d") != current_day: - trigger2cis = dict() - trigger2completed = dict() - current_day = datetime.datetime.today().strftime("%Y-%m-%d") - - if i == 360 or i == 0: - i = 0 - try: - triggers = CITypeTrigger.get_by(to_dict=False) + try: + db.session.remove() + if datetime.datetime.today().strftime("%Y-%m-%d") != current_day: + trigger2cis = dict() + trigger2completed = dict() + current_day = datetime.datetime.today().strftime("%Y-%m-%d") + + if i == 3 or i == 0: + i = 0 + triggers = CITypeTrigger.get_by(to_dict=False, __func_isnot__key_attr_id=None) for trigger in triggers: - ready_cis = CITypeTriggerManager.waiting_cis(trigger) + try: + ready_cis = CITriggerManager.waiting_cis(trigger) + except Exception as e: + print(e) + continue + if trigger.id not in trigger2cis: trigger2cis[trigger.id] = (trigger, ready_cis) else: cur = trigger2cis[trigger.id] cur_ci_ids = {i.ci_id for i in cur[1]} - trigger2cis[trigger.id] = (trigger, cur[1] + [i for i in ready_cis if i.ci_id not in cur_ci_ids - and i.ci_id not in trigger2completed[trigger.id]]) + trigger2cis[trigger.id] = ( + trigger, cur[1] + [i for i in ready_cis if i.ci_id not in cur_ci_ids + and i.ci_id not in trigger2completed.get(trigger.id, {})]) + + for tid in trigger2cis: + trigger, cis = trigger2cis[tid] + for ci in copy.deepcopy(cis): + if CITriggerManager.trigger_notify(trigger, ci): + trigger2completed.setdefault(trigger.id, set()).add(ci.ci_id) + + for _ci in cis: + if _ci.ci_id == ci.ci_id: + cis.remove(_ci) + + i += 1 + time.sleep(10) + except Exception as e: + import traceback + print(traceback.format_exc()) + current_app.logger.error("cmdb trigger exception: {}".format(e)) + time.sleep(60) - except Exception as e: - print(e) - for tid in trigger2cis: - trigger, cis = trigger2cis[tid] - for ci in copy.deepcopy(cis): - if CITypeTriggerManager.trigger_notify(trigger, ci): - trigger2completed.setdefault(trigger.id, set()).add(ci.ci_id) +@click.command() +@with_appcontext +def cmdb_index_table_upgrade(): + """ + Migrate data from tables c_value_integers, c_value_floats, and c_value_datetime + """ + for attr in Attribute.get_by(to_dict=False): + if attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON} and not attr.is_index: + attr.update(is_index=True) + AttributeCache.clean(attr) + + from api.models.cmdb import CIValueInteger, CIIndexValueInteger + from api.models.cmdb import CIValueFloat, CIIndexValueFloat + from api.models.cmdb import CIValueDateTime, CIIndexValueDateTime + + for i in CIValueInteger.get_by(to_dict=False): + CIIndexValueInteger.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False) + i.delete(commit=False) + db.session.commit() + + for i in CIValueFloat.get_by(to_dict=False): + CIIndexValueFloat.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False) + i.delete(commit=False) + db.session.commit() + + for i in CIValueDateTime.get_by(to_dict=False): + CIIndexValueDateTime.create(ci_id=i.ci_id, attr_id=i.attr_id, value=i.value, commit=False) + i.delete(commit=False) + db.session.commit() + + +def valid_address(address): + if not address: + return False + + if not address.startswith(("http://127.0.0.1", "https://127.0.0.1")): + response = { + "message": "Address should start with http://127.0.0.1 or https://127.0.0.1", + "status": "failed" + } + KeyManage.print_response(response) + return False + return True + + +@click.command() +@click.option( + '-a', + '--address', + help='inner cmdb api, http://127.0.0.1:8000', +) +@with_appcontext +def cmdb_inner_secrets_init(address): + """ + init inner secrets for password feature + """ + res, ok = KeyManage(backend=InnerKVManger).init() + if not ok: + if res.get("status") == "failed": + KeyManage.print_response(res) + return + + token = res.get("details", {}).get("root_token", "") + if valid_address(address): + token = current_app.config.get("INNER_TRIGGER_TOKEN", "") if not token else token + if not token: + token = click.prompt(f'Enter root token', hide_input=True, confirmation_prompt=False) + assert token is not None + resp = requests.post("{}/api/v0.1/secrets/auto_seal".format(address.strip("/")), + headers={"Inner-Token": token}) + if resp.status_code == 200: + KeyManage.print_response(resp.json()) + else: + KeyManage.print_response({"message": resp.text or resp.status_code, "status": "failed"}) + else: + KeyManage.print_response(res) + + +@click.command() +@click.option( + '-a', + '--address', + help='inner cmdb api, http://127.0.0.1:8000', + required=True, +) +@with_appcontext +def cmdb_inner_secrets_unseal(address): + """ + unseal the secrets feature + """ + if not valid_address(address): + return + address = "{}/api/v0.1/secrets/unseal".format(address.strip("/")) + for i in range(global_key_threshold): + token = click.prompt(f'Enter unseal token {i + 1}', hide_input=True, confirmation_prompt=False) + assert token is not None + resp = requests.post(address, headers={"Unseal-Token": token}) + if resp.status_code == 200: + KeyManage.print_response(resp.json()) + if resp.json().get("status") in ["success", "skip"]: + return + else: + KeyManage.print_response({"message": resp.status_code, "status": "failed"}) + return + - for _ci in cis: - if _ci.ci_id == ci.ci_id: - cis.remove(_ci) +@click.command() +@click.option( + '-a', + '--address', + help='inner cmdb api, http://127.0.0.1:8000', + required=True, +) +@click.option( + '-k', + '--token', + help='root token', + prompt=True, + hide_input=True, +) +@with_appcontext +def cmdb_inner_secrets_seal(address, token): + """ + seal the secrets feature + """ + assert address is not None + assert token is not None + if not valid_address(address): + return + address = "{}/api/v0.1/secrets/seal".format(address.strip("/")) + resp = requests.post(address, headers={ + "Inner-Token": token, + }) + if resp.status_code == 200: + KeyManage.print_response(resp.json()) + else: + KeyManage.print_response({"message": resp.status_code, "status": "failed"}) + + +@click.command() +@with_appcontext +def cmdb_password_data_migrate(): + """ + Migrate CI password data, version >= v2.3.6 + """ + from api.models.cmdb import CIIndexValueText + from api.models.cmdb import CIValueText + from api.lib.secrets.inner import InnerCrypt + from api.lib.secrets.vault import VaultClient + + attrs = Attribute.get_by(to_dict=False) + for attr in attrs: + if attr.is_password: + + value_table = CIIndexValueText if attr.is_index else CIValueText + + failed = False + for i in value_table.get_by(attr_id=attr.id, to_dict=False): + if current_app.config.get("SECRETS_ENGINE", 'inner') == 'inner': + _, status = InnerCrypt().decrypt(i.value) + if status: + continue + + encrypt_value, status = InnerCrypt().encrypt(i.value) + if status: + CIValueText.create(ci_id=i.ci_id, attr_id=attr.id, value=encrypt_value) + else: + failed = True + continue + elif current_app.config.get("SECRETS_ENGINE") == 'vault': + if i.value == '******': + continue + + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + try: + vault.update("/{}/{}".format(i.ci_id, i.attr_id), dict(v=i.value)) + except Exception as e: + print('save password to vault failed: {}'.format(e)) + failed = True + continue + else: + continue + + i.delete() + + if not failed and attr.is_index: + attr.update(is_index=False) + + +@click.command() +@with_appcontext +def cmdb_agent_init(): + """ + Initialize the agent's permissions and obtain the key and secret + """ + + from api.models.acl import User + + user = User.get_by(username="cmdb_agent", first=True, to_dict=False) + if user is None: + click.echo( + click.style('user cmdb_agent does not exist, please use flask add-user to create it first', fg='red')) + return + + # grant + _app = AppCache.get('cmdb') or App.create(name='cmdb') + app_id = _app.id + + ci_types = CIType.get_by(to_dict=False) + resource_type_id = ResourceType.get_by(name=ResourceTypeEnum.CI, first=True, to_dict=False).id + for ci_type in ci_types: + try: + ResourceCRUD.add(ci_type.name, resource_type_id, app_id) + except AbortException: + pass + + ACLManager().grant_resource_to_role(ci_type.name, + "cmdb_agent", + ResourceTypeEnum.CI, + [PermEnum.READ, PermEnum.UPDATE, PermEnum.ADD, PermEnum.DELETE]) - i += 1 - time.sleep(10) + click.echo("Key : {}".format(click.style(user.key, bg='red'))) + click.echo("Secret: {}".format(click.style(user.secret, bg='red'))) diff --git a/cmdb-api/api/commands/click_common_setting.py b/cmdb-api/api/commands/click_common_setting.py new file mode 100644 index 00000000..a1f325e1 --- /dev/null +++ b/cmdb-api/api/commands/click_common_setting.py @@ -0,0 +1,301 @@ +import click +from flask import current_app +from flask.cli import with_appcontext +from werkzeug.datastructures import MultiDict + +from api.lib.common_setting.acl import ACLManager +from api.lib.common_setting.employee import EmployeeAddForm +from api.lib.common_setting.resp_format import ErrFormat +from api.models.common_setting import Employee, Department + + +class InitEmployee(object): + + def __init__(self): + self.log = current_app.logger + + def import_user_from_acl(self): + """ + Import users from ACL + """ + + InitDepartment().init() + acl = ACLManager('acl') + user_list = acl.get_all_users() + + username_list = [e['username'] for e in Employee.get_by()] + + for user in user_list: + acl_uid = user['uid'] + block = 1 if user['block'] else 0 + acl_rid = self.get_rid_by_uid(acl_uid) + if user['username'] in username_list: + existed = Employee.get_by(first=True, username=user['username'], to_dict=False) + if existed: + existed.update( + acl_uid=acl_uid, + acl_rid=acl_rid, + block=block, + ) + continue + try: + form = EmployeeAddForm(MultiDict(user)) + if not form.validate(): + raise Exception( + ','.join(['{}: {}'.format(filed, ','.join(msg)) for filed, msg in form.errors.items()])) + data = form.data + data['acl_uid'] = acl_uid + data['acl_rid'] = acl_rid + data['block'] = block + data.pop('password') + Employee.create( + **data + ) + except Exception as e: + self.log.error(ErrFormat.acl_import_user_failed.format(user['username'], str(e))) + self.log.error(e) + + @staticmethod + def get_rid_by_uid(uid): + from api.models.acl import Role + role = Role.get_by(first=True, uid=uid) + return role['id'] if role is not None else 0 + + +class InitDepartment(object): + def __init__(self): + self.log = current_app.logger + + def init(self): + self.init_wide_company() + + @staticmethod + def hard_delete(department_id, department_name): + existed_deleted_list = Department.query.filter( + Department.department_name == department_name, + Department.department_id == department_id, + Department.deleted == 1, + ).all() + for existed in existed_deleted_list: + existed.delete() + + @staticmethod + def get_department(department_name): + return Department.query.filter( + Department.department_name == department_name, + Department.deleted == 0, + ).first() + + def run(self, department_id, department_name, department_parent_id): + self.hard_delete(department_id, department_name) + + res = self.get_department(department_name) + if res: + if res.department_id == department_id: + return + else: + res.update( + department_id=department_id, + department_parent_id=department_parent_id, + ) + return + + Department.create( + department_id=department_id, + department_name=department_name, + department_parent_id=department_parent_id, + ) + new_d = self.get_department(department_name) + + if new_d.department_id != department_id: + new_d.update( + department_id=department_id, + department_parent_id=department_parent_id, + ) + self.log.info(f"init {department_name} success.") + + def run_common(self, department_id, department_name, department_parent_id): + try: + self.run(department_id, department_name, department_parent_id) + except Exception as e: + current_app.logger.error(f"init {department_name} err:") + current_app.logger.error(e) + raise Exception(e) + + def init_wide_company(self): + department_id = 0 + department_name = '全公司' + department_parent_id = -1 + + self.run_common(department_id, department_name, department_parent_id) + + @staticmethod + def create_acl_role_with_department(): + acl = ACLManager('acl') + role_name_map = {role['name']: role for role in acl.get_all_roles()} + + d_list = Department.query.filter( + Department.deleted == 0, Department.department_parent_id != -1).all() + for department in d_list: + if department.acl_rid > 0: + continue + + role = role_name_map.get(department.department_name) + if not role: + payload = { + 'app_id': 'acl', + 'name': department.department_name, + } + role = acl.create_role(payload) + + acl_rid = role.get('id') if role else 0 + + department.update( + acl_rid=acl_rid + ) + info = f"update department acl_rid: {acl_rid}" + current_app.logger.info(info) + + def init_backend_resource(self): + acl = self.check_app('backend') + resources_types = acl.get_all_resources_types() + + perms = ['read', 'grant', 'delete', 'update'] + + acl_rid = self.get_admin_user_rid() + + results = list(filter(lambda t: t['name'] == '操作权限', resources_types['groups'])) + if len(results) == 0: + payload = dict( + app_id=acl.app_name, + name='操作权限', + description='', + perms=perms + ) + resource_type = acl.create_resources_type(payload) + else: + resource_type = results[0] + resource_type_id = resource_type['id'] + existed_perms = resources_types.get('id2perms', {}).get(resource_type_id, []) + existed_perms = [p['name'] for p in existed_perms] + new_perms = [] + for perm in perms: + if perm not in existed_perms: + new_perms.append(perm) + if len(new_perms) > 0: + resource_type['perms'] = existed_perms + new_perms + acl.update_resources_type(resource_type_id, resource_type) + + resource_list = acl.get_resource_by_type(None, None, resource_type['id']) + + for name in ['公司信息', '公司架构', '通知设置']: + target = list(filter(lambda r: r['name'] == name, resource_list)) + if len(target) == 0: + payload = dict( + type_id=resource_type['id'], + app_id=acl.app_name, + name=name, + ) + resource = acl.create_resource(payload) + else: + resource = target[0] + + if acl_rid > 0: + acl.grant_resource(acl_rid, resource['id'], perms) + + @staticmethod + def check_app(app_name): + acl = ACLManager(app_name) + payload = dict( + name=app_name, + description=app_name + ) + app = acl.validate_app() + if not app: + acl.create_app(payload) + return acl + + @staticmethod + def get_admin_user_rid(): + admin = Employee.get_by(first=True, username='admin', to_dict=False) + return admin.acl_rid if admin else 0 + + +@click.command() +@with_appcontext +def init_import_user_from_acl(): + """ + Import users from ACL + """ + InitEmployee().import_user_from_acl() + + +@click.command() +@with_appcontext +def init_department(): + """ + Department initialization + """ + cli = InitDepartment() + cli.init_wide_company() + cli.create_acl_role_with_department() + cli.init_backend_resource() + + +@click.command() +@with_appcontext +def common_check_new_columns(): + """ + add new columns to tables + """ + from api.extensions import db + from sqlalchemy import inspect, text + + def get_model_by_table_name(_table_name): + registry = getattr(db.Model, 'registry', None) + class_registry = getattr(registry, '_class_registry', None) + for _model in class_registry.values(): + if hasattr(_model, '__tablename__') and _model.__tablename__ == _table_name: + return _model + return None + + def add_new_column(target_table_name, new_column): + column_type = new_column.type.compile(engine.dialect) + default_value = new_column.default.arg if new_column.default else None + + sql = "ALTER TABLE " + target_table_name + " ADD COLUMN " + new_column.name + " " + column_type + if new_column.comment: + sql += f" comment '{new_column.comment}'" + + if column_type == 'JSON': + pass + elif default_value: + if column_type.startswith('VAR') or column_type.startswith('Text'): + if default_value is None or len(default_value) == 0: + pass + else: + sql += f" DEFAULT {default_value}" + + sql = text(sql) + db.session.execute(sql) + + engine = db.get_engine() + inspector = inspect(engine) + table_names = inspector.get_table_names() + for table_name in table_names: + existed_columns = inspector.get_columns(table_name) + existed_column_name_list = [c['name'] for c in existed_columns] + + model = get_model_by_table_name(table_name) + if model is None: + continue + + model_columns = getattr(getattr(getattr(model, '__table__'), 'columns'), '_all_columns') + for column in model_columns: + if column.name not in existed_column_name_list: + try: + add_new_column(table_name, column) + current_app.logger.info(f"add new column [{column.name}] in table [{table_name}] success.") + except Exception as e: + current_app.logger.error(f"add new column [{column.name}] in table [{table_name}] err:") + current_app.logger.error(e) diff --git a/cmdb-api/api/commands/common.py b/cmdb-api/api/commands/common.py index 1d10f1cf..6313ef61 100644 --- a/cmdb-api/api/commands/common.py +++ b/cmdb-api/api/commands/common.py @@ -84,66 +84,6 @@ def clean(): os.remove(full_pathname) -@click.command() -@click.option("--url", default=None, help="Url to test (ex. /static/image.png)") -@click.option( - "--order", default="rule", help="Property on Rule to order by (default: rule)" -) -@with_appcontext -def urls(url, order): - """Display all of the url matching routes for the project. - - Borrowed from Flask-Script, converted to use Click. - """ - rows = [] - column_headers = ("Rule", "Endpoint", "Arguments") - - if url: - try: - rule, arguments = current_app.url_map.bind("localhost").match( - url, return_rule=True - ) - rows.append((rule.rule, rule.endpoint, arguments)) - column_length = 3 - except (NotFound, MethodNotAllowed) as e: - rows.append(("<{}>".format(e), None, None)) - column_length = 1 - else: - rules = sorted( - current_app.url_map.iter_rules(), key=lambda rule: getattr(rule, order) - ) - for rule in rules: - rows.append((rule.rule, rule.endpoint, None)) - column_length = 2 - - str_template = "" - table_width = 0 - - if column_length >= 1: - max_rule_length = max(len(r[0]) for r in rows) - max_rule_length = max_rule_length if max_rule_length > 4 else 4 - str_template += "{:" + str(max_rule_length) + "}" - table_width += max_rule_length - - if column_length >= 2: - max_endpoint_length = max(len(str(r[1])) for r in rows) - max_endpoint_length = max_endpoint_length if max_endpoint_length > 8 else 8 - str_template += " {:" + str(max_endpoint_length) + "}" - table_width += 2 + max_endpoint_length - - if column_length >= 3: - max_arguments_length = max(len(str(r[2])) for r in rows) - max_arguments_length = max_arguments_length if max_arguments_length > 9 else 9 - str_template += " {:" + str(max_arguments_length) + "}" - table_width += 2 + max_arguments_length - - click.echo(str_template.format(*column_headers[:column_length])) - click.echo("-" * table_width) - - for row in rows: - click.echo(str_template.format(*row[:column_length])) - - @click.command() @with_appcontext def db_setup(): diff --git a/cmdb-api/api/commands/init_common_setting.py b/cmdb-api/api/commands/init_common_setting.py deleted file mode 100644 index ddc67bcc..00000000 --- a/cmdb-api/api/commands/init_common_setting.py +++ /dev/null @@ -1,164 +0,0 @@ -import click -from flask import current_app -from flask.cli import with_appcontext -from werkzeug.datastructures import MultiDict - -from api.lib.common_setting.acl import ACLManager -from api.lib.common_setting.employee import EmployeeAddForm -from api.lib.common_setting.resp_format import ErrFormat -from api.models.common_setting import Employee, Department - - -class InitEmployee(object): - """ - 初始化员工 - """ - - def __init__(self): - self.log = current_app.logger - - def import_user_from_acl(self): - """ - 从ACL导入用户 - """ - - acl = ACLManager('acl') - user_list = acl.get_all_users() - - username_list = [e['username'] for e in Employee.get_by()] - - for user in user_list: - if user['username'] in username_list: - continue - try: - form = EmployeeAddForm(MultiDict(user)) - if not form.validate(): - raise Exception( - ','.join(['{}: {}'.format(filed, ','.join(msg)) for filed, msg in form.errors.items()])) - data = form.data - data['acl_uid'] = user['uid'] - data['block'] = 1 if user['block'] else 0 - data.pop('password') - Employee.create( - **data - ) - except Exception as e: - self.log.error(ErrFormat.acl_import_user_failed.format(user['username'], str(e))) - self.log.error(e) - - -class InitDepartment(object): - def __init__(self): - self.log = current_app.logger - - def init(self): - self.init_wide_company() - - def hard_delete(self, department_id, department_name): - existed_deleted_list = Department.query.filter( - Department.department_name == department_name, - Department.department_id == department_id, - Department.deleted == 1, - ).all() - for existed in existed_deleted_list: - existed.delete() - - def get_department(self, department_name): - return Department.query.filter( - Department.department_name == department_name, - Department.deleted == 0, - ).order_by(Department.created_at.asc()).first() - - def run(self, department_id, department_name, department_parent_id): - self.hard_delete(department_id, department_name) - - res = self.get_department(department_name) - if res: - if res.department_id == department_id: - return - else: - new_d = res.update( - department_id=department_id, - department_parent_id=department_parent_id, - ) - return - - Department.create( - department_id=department_id, - department_name=department_name, - department_parent_id=department_parent_id, - ) - new_d = self.get_department(department_name) - - if new_d.department_id != department_id: - new_d = new_d.update( - department_id=department_id, - department_parent_id=department_parent_id, - ) - self.log.info(f"初始化 {department_name} 部门成功.") - - def run_common(self, department_id, department_name, department_parent_id): - try: - self.run(department_id, department_name, department_parent_id) - except Exception as e: - current_app.logger.error(f"init {department_name} err:") - current_app.logger.error(e) - raise Exception(e) - - def init_wide_company(self): - """ - 创建 id 0, name 全公司 的部门 - """ - department_id = 0 - department_name = '全公司' - department_parent_id = -1 - - self.run_common(department_id, department_name, department_parent_id) - - def create_acl_role_with_department(self): - """ - 当前所有部门,在ACL创建 role - """ - acl = ACLManager('acl') - role_name_map = {role['name']: role for role in acl.get_all_roles()} - - d_list = Department.query.filter( - Department.deleted == 0, Department.department_parent_id != -1).all() - for department in d_list: - if department.acl_rid > 0: - continue - - role = role_name_map.get(department.department_name) - if role is None: - payload = { - 'app_id': 'acl', - 'name': department.department_name, - } - role = acl.create_role(payload) - - acl_rid = role.get('id') if role else 0 - - department.update( - acl_rid=acl_rid - ) - info = f"update department acl_rid: {acl_rid}" - current_app.logger.info(info) - - -@click.command() -@with_appcontext -def init_import_user_from_acl(): - """ - 从ACL导入用户 - """ - InitEmployee().import_user_from_acl() - - -@click.command() -@with_appcontext -def init_department(): - """ - 初始化 部门 - """ - InitDepartment().init() - InitDepartment().create_acl_role_with_department() diff --git a/cmdb-api/api/extensions.py b/cmdb-api/api/extensions.py index f540c21b..2c3ff5fd 100644 --- a/cmdb-api/api/extensions.py +++ b/cmdb-api/api/extensions.py @@ -9,6 +9,7 @@ from flask_migrate import Migrate from flask_sqlalchemy import SQLAlchemy +from api.lib.secrets.inner import KeyManage from api.lib.utils import ESHandler from api.lib.utils import RedisHandler @@ -21,3 +22,4 @@ cors = CORS(supports_credentials=True) rd = RedisHandler() es = ESHandler() +inner_secrets = KeyManage() diff --git a/cmdb-api/api/lib/cmdb/attribute.py b/cmdb-api/api/lib/cmdb/attribute.py index 97b2aaa7..817bac26 100644 --- a/cmdb-api/api/lib/cmdb/attribute.py +++ b/cmdb-api/api/lib/cmdb/attribute.py @@ -1,15 +1,20 @@ -# -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- -import requests from flask import abort from flask import current_app -from flask import g from flask import session +from flask_login import current_user from api.extensions import db from api.lib.cmdb.cache import AttributeCache +from api.lib.cmdb.cache import CITypeAttributesCache +from api.lib.cmdb.cache import CITypeCache +from api.lib.cmdb.const import BUILTIN_KEYWORDS from api.lib.cmdb.const import CITypeOperateType -from api.lib.cmdb.const import ResourceTypeEnum, RoleEnum, PermEnum +from api.lib.cmdb.const import CMDB_QUEUE +from api.lib.cmdb.const import PermEnum +from api.lib.cmdb.const import ResourceTypeEnum +from api.lib.cmdb.const import RoleEnum from api.lib.cmdb.const import ValueTypeEnum from api.lib.cmdb.history import CITypeHistoryManager from api.lib.cmdb.resp_format import ErrFormat @@ -17,7 +22,9 @@ from api.lib.decorator import kwargs_required from api.lib.perm.acl.acl import is_app_admin from api.lib.perm.acl.acl import validate_permission +from api.lib.webhook import webhook_request from api.models.cmdb import Attribute +from api.models.cmdb import CIType from api.models.cmdb import CITypeAttribute from api.models.cmdb import CITypeAttributeGroupItem from api.models.cmdb import PreferenceShowAttributes @@ -33,15 +40,11 @@ def __init__(self): pass @staticmethod - def _get_choice_values_from_web_hook(choice_web_hook): - url = choice_web_hook.get('url') - ret_key = choice_web_hook.get('ret_key') - headers = choice_web_hook.get('headers') or {} - payload = choice_web_hook.get('payload') or {} - method = choice_web_hook.get('method', 'GET').lower() + def _get_choice_values_from_webhook(choice_webhook, payload=None): + ret_key = choice_webhook.get('ret_key') try: - res = getattr(requests, method)(url, headers=headers, data=payload).json() + res = webhook_request(choice_webhook, payload or {}).json() if ret_key: ret_key_list = ret_key.strip().split("##") for key in ret_key_list[:-1]: @@ -53,52 +56,92 @@ def _get_choice_values_from_web_hook(choice_web_hook): return [[i, {}] for i in (res.get(ret_key_list[-1]) or [])] except Exception as e: - current_app.logger.error(str(e)) + current_app.logger.error("get choice values failed: {}".format(e)) return [] + @staticmethod + def _get_choice_values_from_other(choice_other): + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + + if choice_other.get('type_ids'): + type_ids = choice_other.get('type_ids') + attr_id = choice_other.get('attr_id') + other_filter = choice_other.get('filter') or '' + + query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter) + s = search(query, fl=[str(attr_id)], facet=[str(attr_id)], count=1) + try: + _, _, _, _, _, facet = s.search() + return [[i[0], {}] for i in (list(facet.values()) or [[]])[0]] + except SearchError as e: + current_app.logger.error("get choice values from other ci failed: {}".format(e)) + return [] + + elif choice_other.get('script'): + try: + x = compile(choice_other['script'], '', "exec") + local_ns = {} + exec(x, {}, local_ns) + res = local_ns['ChoiceValue']().values() or [] + return [[i, {}] for i in res] + except Exception as e: + current_app.logger.error("get choice values from script: {}".format(e)) + return [] + @classmethod - def get_choice_values(cls, attr_id, value_type, choice_web_hook, choice_web_hook_parse=True): - if choice_web_hook and isinstance(choice_web_hook, dict) and choice_web_hook_parse: - return cls._get_choice_values_from_web_hook(choice_web_hook) - elif choice_web_hook and not choice_web_hook_parse: - return [] + def get_choice_values(cls, attr_id, value_type, choice_web_hook, choice_other, + choice_web_hook_parse=True, choice_other_parse=True): + if choice_web_hook: + if choice_web_hook_parse and isinstance(choice_web_hook, dict): + return cls._get_choice_values_from_webhook(choice_web_hook) + else: + return [] + elif choice_other: + if choice_other_parse and isinstance(choice_other, dict): + return cls._get_choice_values_from_other(choice_other) + else: + return [] choice_table = ValueTypeMap.choice.get(value_type) + if not choice_table: + return [] choice_values = choice_table.get_by(fl=["value", "option"], attr_id=attr_id) - return [[choice_value['value'], choice_value['option']] for choice_value in choice_values] + return [[ValueTypeMap.serialize[value_type](choice_value['value']), choice_value['option']] + for choice_value in choice_values] @staticmethod def add_choice_values(_id, value_type, choice_values): choice_table = ValueTypeMap.choice.get(value_type) + if choice_table is None: + return - db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() - db.session.flush() - choice_values = choice_values - for v, option in choice_values: - table = choice_table(attr_id=_id, value=v, option=option) + choice_table.get_by(attr_id=_id, only_query=True).delete() - db.session.add(table) + for v, option in choice_values: + choice_table.create(attr_id=_id, value=v, option=option, commit=False) try: db.session.flush() - except: + except Exception as e: + current_app.logger.warning("add choice values failed: {}".format(e)) return abort(400, ErrFormat.invalid_choice_values) @staticmethod def _del_choice_values(_id, value_type): choice_table = ValueTypeMap.choice.get(value_type) - db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() + choice_table and choice_table.get_by(attr_id=_id, only_query=True).delete() db.session.flush() @classmethod def search_attributes(cls, name=None, alias=None, page=1, page_size=None): """ - :param name: - :param alias: - :param page: - :param page_size: + :param name: + :param alias: + :param page: + :param page_size: :return: attribute, if name is None, then return all attributes """ if name is not None: @@ -112,8 +155,9 @@ def search_attributes(cls, name=None, alias=None, page=1, page_size=None): attrs = attrs[(page - 1) * page_size:][:page_size] res = list() for attr in attrs: - attr["is_choice"] and attr.update(dict(choice_value=cls.get_choice_values( - attr["id"], attr["value_type"], attr["choice_web_hook"]))) + attr["is_choice"] and attr.update( + dict(choice_value=cls.get_choice_values(attr["id"], attr["value_type"], + attr["choice_web_hook"], attr.get("choice_other")))) attr['is_choice'] and attr.pop('choice_web_hook', None) res.append(attr) @@ -122,30 +166,40 @@ def search_attributes(cls, name=None, alias=None, page=1, page_size=None): def get_attribute_by_name(self, name): attr = Attribute.get_by(name=name, first=True) - if attr and attr["is_choice"]: - attr.update(dict(choice_value=self.get_choice_values( - attr["id"], attr["value_type"], attr["choice_web_hook"]))) + if attr.get("is_choice"): + attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"], + attr["choice_web_hook"], attr.get("choice_other")) + return attr def get_attribute_by_alias(self, alias): attr = Attribute.get_by(alias=alias, first=True) - if attr and attr["is_choice"]: - attr.update(dict(choice_value=self.get_choice_values( - attr["id"], attr["value_type"], attr["choice_web_hook"]))) + if attr.get("is_choice"): + attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"], + attr["choice_web_hook"], attr.get("choice_other")) + return attr def get_attribute_by_id(self, _id): attr = Attribute.get_by_id(_id).to_dict() - if attr and attr["is_choice"]: - attr.update(dict(choice_value=self.get_choice_values( - attr["id"], attr["value_type"], attr["choice_web_hook"]))) + if attr.get("is_choice"): + attr["choice_value"] = self.get_choice_values(attr["id"], attr["value_type"], + attr["choice_web_hook"], attr.get("choice_other")) + return attr - def get_attribute(self, key, choice_web_hook_parse=True): + def get_attribute(self, key, choice_web_hook_parse=True, choice_other_parse=True): attr = AttributeCache.get(key).to_dict() - if attr and attr["is_choice"]: - attr.update(dict(choice_value=self.get_choice_values( - attr["id"], attr["value_type"], attr["choice_web_hook"])), choice_web_hook_parse=choice_web_hook_parse) + if attr.get("is_choice"): + attr["choice_value"] = self.get_choice_values( + attr["id"], + attr["value_type"], + attr["choice_web_hook"], + attr.get("choice_other"), + choice_web_hook_parse=choice_web_hook_parse, + choice_other_parse=choice_other_parse, + ) + return attr @staticmethod @@ -153,16 +207,40 @@ def can_create_computed_attribute(): if RoleEnum.CONFIG not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin('cmdb'): return abort(403, ErrFormat.role_required.format(RoleEnum.CONFIG)) + @classmethod + def calc_computed_attribute(cls, attr_id): + """ + calculate computed attribute for all ci + :param attr_id: + :return: + """ + cls.can_create_computed_attribute() + + from api.tasks.cmdb import calc_computed_attribute + + calc_computed_attribute.apply_async(args=(attr_id, current_user.uid), queue=CMDB_QUEUE) + @classmethod @kwargs_required("name") def add(cls, **kwargs): choice_value = kwargs.pop("choice_value", []) kwargs.pop("is_choice", None) - is_choice = True if choice_value or kwargs.get('choice_web_hook') else False + is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False name = kwargs.pop("name") - if name in {'id', '_id', 'ci_id', 'type', '_type', 'ci_type'}: + if name in BUILTIN_KEYWORDS: return abort(400, ErrFormat.attribute_name_cannot_be_builtin) + + while kwargs.get('choice_other'): + if isinstance(kwargs['choice_other'], dict): + if kwargs['choice_other'].get('script'): + break + + if kwargs['choice_other'].get('type_ids') and kwargs['choice_other'].get('attr_id'): + break + + return abort(400, ErrFormat.attribute_choice_other_invalid) + alias = kwargs.pop("alias", "") alias = name if not alias else alias Attribute.get_by(name=name, first=True) and abort(400, ErrFormat.attribute_name_duplicate.format(name)) @@ -172,11 +250,13 @@ def add(cls, **kwargs): kwargs.get('is_computed') and cls.can_create_computed_attribute() + kwargs.get('choice_other') and kwargs['choice_other'].get('script') and cls.can_create_computed_attribute() + attr = Attribute.create(flush=True, name=name, alias=alias, is_choice=is_choice, - uid=g.user.uid, + uid=current_user.uid, **kwargs) if choice_value: @@ -210,6 +290,11 @@ def add(cls, **kwargs): return attr.id + @staticmethod + def _clean_ci_type_attributes_cache(attr_id): + for i in CITypeAttribute.get_by(attr_id=attr_id, to_dict=False): + CITypeAttributesCache.clean(i.type_id) + @staticmethod def _change_index(attr, old, new): from api.lib.cmdb.utils import TableMap @@ -220,11 +305,11 @@ def _change_index(attr, old, new): new_table = TableMap(attr=attr, is_index=new).table ci_ids = [] - for i in db.session.query(old_table).filter(getattr(old_table, 'attr_id') == attr.id): + for i in old_table.get_by(attr_id=attr.id, to_dict=False): new_table.create(ci_id=i.ci_id, attr_id=attr.id, value=i.value, flush=True) ci_ids.append(i.ci_id) - db.session.query(old_table).filter(getattr(old_table, 'attr_id') == attr.id).delete() + old_table.get_by(attr_id=attr.id, only_query=True).delete() try: db.session.commit() @@ -239,7 +324,7 @@ def _change_index(attr, old, new): def _can_edit_attribute(attr): from api.lib.cmdb.ci_type import CITypeManager - if attr.uid == g.user.uid: + if attr.uid == current_user.uid: return True for i in CITypeAttribute.get_by(attr_id=attr.id, to_dict=False): @@ -252,9 +337,6 @@ def _can_edit_attribute(attr): def update(self, _id, **kwargs): attr = Attribute.get_by_id(_id) or abort(404, ErrFormat.attribute_not_found.format("id={}".format(_id))) - if not self._can_edit_attribute(attr): - return abort(403, ErrFormat.cannot_edit_attribute) - if kwargs.get("name"): other = Attribute.get_by(name=kwargs['name'], first=True, to_dict=False) if other and other.id != attr.id: @@ -272,12 +354,22 @@ def update(self, _id, **kwargs): self._change_index(attr, attr.is_index, kwargs['is_index']) + while kwargs.get('choice_other'): + if isinstance(kwargs['choice_other'], dict): + if kwargs['choice_other'].get('script'): + break + + if kwargs['choice_other'].get('type_ids') and kwargs['choice_other'].get('attr_id'): + break + + return abort(400, ErrFormat.attribute_choice_other_invalid) + existed2 = attr.to_dict() - if not existed2['choice_web_hook'] and existed2['is_choice']: - existed2['choice_value'] = self.get_choice_values(attr.id, attr.value_type, attr.choice_web_hook) + if not existed2['choice_web_hook'] and not existed2.get('choice_other') and existed2['is_choice']: + existed2['choice_value'] = self.get_choice_values(attr.id, attr.value_type, None, None) choice_value = kwargs.pop("choice_value", False) - is_choice = True if choice_value or kwargs.get('choice_web_hook') else False + is_choice = True if choice_value or kwargs.get('choice_web_hook') or kwargs.get('choice_other') else False kwargs['is_choice'] = is_choice if kwargs.get('default') and not (isinstance(kwargs['default'], dict) and 'default' in kwargs['default']): @@ -285,11 +377,19 @@ def update(self, _id, **kwargs): kwargs.get('is_computed') and self.can_create_computed_attribute() + is_changed = False + for k in kwargs: + if kwargs[k] != getattr(attr, k, None): + is_changed = True + + if is_changed and not self._can_edit_attribute(attr): + return abort(403, ErrFormat.cannot_edit_attribute) + attr.update(flush=True, filter_none=False, **kwargs) if is_choice and choice_value: self.add_choice_values(attr.id, attr.value_type, choice_value) - elif is_choice: + elif existed2['is_choice']: self._del_choice_values(attr.id, attr.value_type) try: @@ -308,6 +408,8 @@ def update(self, _id, **kwargs): AttributeCache.clean(attr) + self._clean_ci_type_attributes_cache(_id) + return attr.id @staticmethod @@ -315,25 +417,31 @@ def delete(_id): attr = Attribute.get_by_id(_id) or abort(404, ErrFormat.attribute_not_found.format("id={}".format(_id))) name = attr.name - if attr.uid and attr.uid != g.user.uid: + if CIType.get_by(unique_id=attr.id, first=True, to_dict=False) is not None: + return abort(400, ErrFormat.attribute_is_unique_id) + + ref = CITypeAttribute.get_by(attr_id=_id, to_dict=False, first=True) + if ref is not None: + ci_type = CITypeCache.get(ref.type_id) + return abort(400, ErrFormat.attribute_is_ref_by_type.format(ci_type and ci_type.alias or ref.type_id)) + + if attr.uid != current_user.uid and not is_app_admin('cmdb'): return abort(403, ErrFormat.cannot_delete_attribute) if attr.is_choice: choice_table = ValueTypeMap.choice.get(attr.value_type) - db.session.query(choice_table).filter(choice_table.attr_id == _id).delete() # FIXME: session conflict - db.session.flush() - - AttributeCache.clean(attr) + choice_table.get_by(attr_id=_id, only_query=True).delete() attr.soft_delete() - for i in CITypeAttribute.get_by(attr_id=_id, to_dict=False): - i.soft_delete() + AttributeCache.clean(attr) for i in PreferenceShowAttributes.get_by(attr_id=_id, to_dict=False): - i.soft_delete() + i.soft_delete(commit=False) for i in CITypeAttributeGroupItem.get_by(attr_id=_id, to_dict=False): - i.soft_delete() + i.soft_delete(commit=False) + + db.session.commit() return name diff --git a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py index 54a84486..cec57f58 100644 --- a/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py +++ b/cmdb-api/api/lib/cmdb/auto_discovery/auto_discovery.py @@ -5,7 +5,7 @@ from flask import abort from flask import current_app -from flask import g +from flask_login import current_user from sqlalchemy import func from api.extensions import db @@ -36,9 +36,10 @@ def parse_plugin_script(script): attributes = [] try: x = compile(script, '', "exec") - exec(x) - unique_key = locals()['AutoDiscovery']().unique_key - attrs = locals()['AutoDiscovery']().attributes() or [] + local_ns = {} + exec(x, {}, local_ns) + unique_key = local_ns['AutoDiscovery']().unique_key + attrs = local_ns['AutoDiscovery']().attributes() or [] except Exception as e: return abort(400, str(e)) @@ -156,7 +157,7 @@ def get(cls, ci_id, oneagent_id, last_update_at=None): continue if isinstance(rule.get("extra_option"), dict) and rule['extra_option'].get('secret'): - if not (g.user.username == "cmdb_agent" or g.user.uid == rule['uid']): + if not (current_user.username == "cmdb_agent" or current_user.uid == rule['uid']): rule['extra_option'].pop('secret', None) else: rule['extra_option']['secret'] = AESCrypto.decrypt(rule['extra_option']['secret']) @@ -213,7 +214,7 @@ def __valid_exec_target(agent_id, query_expr): agent_id = agent_id.strip() q = "op_duty:{0},-rd_duty:{0},oneagent_id:{1}" - s = search(q.format(g.user.username, agent_id.strip())) + s = search(q.format(current_user.username, agent_id.strip())) try: response, _, _, _, _, _ = s.search() if response: @@ -222,7 +223,7 @@ def __valid_exec_target(agent_id, query_expr): current_app.logger.warning(e) return abort(400, str(e)) - s = search(q.format(g.user.nickname, agent_id.strip())) + s = search(q.format(current_user.nickname, agent_id.strip())) try: response, _, _, _, _, _ = s.search() if response: @@ -240,9 +241,10 @@ def __valid_exec_target(agent_id, query_expr): try: response, _, _, _, _, _ = s.search() for i in response: - if g.user.username not in (i.get('rd_duty') or []) and g.user.username not in \ - (i.get('op_duty') or []) and g.user.nickname not in (i.get('rd_duty') or []) and \ - g.user.nickname not in (i.get('op_duty') or []): + if (current_user.username not in (i.get('rd_duty') or []) and + current_user.username not in (i.get('op_duty') or []) and + current_user.nickname not in (i.get('rd_duty') or []) and + current_user.nickname not in (i.get('op_duty') or [])): return abort(403, ErrFormat.adt_target_expr_no_permission.format( i.get("{}_name".format(i.get('ci_type'))))) except SearchError as e: @@ -270,7 +272,7 @@ def _can_add(self, **kwargs): if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): kwargs['extra_option']['secret'] = AESCrypto.encrypt(kwargs['extra_option']['secret']) - kwargs['uid'] = g.user.uid + kwargs['uid'] = current_user.uid return kwargs @@ -281,7 +283,7 @@ def _can_update(self, **kwargs): self.__valid_exec_target(kwargs.get('agent_id'), kwargs.get('query_expr')) if isinstance(kwargs.get('extra_option'), dict) and kwargs['extra_option'].get('secret'): - if g.user.uid != existed.uid: + if current_user.uid != existed.uid: return abort(403, ErrFormat.adt_secret_no_permission) return existed @@ -453,10 +455,12 @@ def accept(cls, adc, adc_id=None, nickname=None): relation_adts = AutoDiscoveryCIType.get_by(type_id=adt.type_id, adr_id=None, to_dict=False) for r_adt in relation_adts: - if r_adt.relation and ci_id is not None: - ad_key, cmdb_key = None, {} - for ad_key in r_adt.relation: - cmdb_key = r_adt.relation[ad_key] + if not r_adt.relation or ci_id is None: + continue + for ad_key in r_adt.relation: + if not adc.instance.get(ad_key): + continue + cmdb_key = r_adt.relation[ad_key] query = "_type:{},{}:{}".format(cmdb_key.get('type_name'), cmdb_key.get('attr_name'), adc.instance.get(ad_key)) s = search(query) @@ -476,7 +480,10 @@ def accept(cls, adc, adc_id=None, nickname=None): except: pass - adc.update(is_accept=True, accept_by=nickname or g.user.nickname, accept_time=datetime.datetime.now()) + adc.update(is_accept=True, + accept_by=nickname or current_user.nickname, + accept_time=datetime.datetime.now(), + ci_id=ci_id) class AutoDiscoveryHTTPManager(object): diff --git a/cmdb-api/api/lib/cmdb/cache.py b/cmdb-api/api/lib/cmdb/cache.py index 68f0905b..c19812fe 100644 --- a/cmdb-api/api/lib/cmdb/cache.py +++ b/cmdb-api/api/lib/cmdb/cache.py @@ -2,14 +2,11 @@ from __future__ import unicode_literals -import requests from flask import current_app from api.extensions import cache -from api.extensions import db from api.lib.cmdb.custom_dashboard import CustomDashboardManager from api.models.cmdb import Attribute -from api.models.cmdb import CI from api.models.cmdb import CIType from api.models.cmdb import CITypeAttribute from api.models.cmdb import RelationType @@ -34,6 +31,7 @@ def get(cls, key): attr = attr or Attribute.get_by(alias=key, first=True, to_dict=False) if attr is not None: cls.set(attr) + return attr @classmethod @@ -67,6 +65,7 @@ def get(cls, key): ct = ct or CIType.get_by(alias=key, first=True, to_dict=False) if ct is not None: cls.set(ct) + return ct @classmethod @@ -98,6 +97,7 @@ def get(cls, key): ct = RelationType.get_by(name=key, first=True, to_dict=False) or RelationType.get_by_id(key) if ct is not None: cls.set(ct) + return ct @classmethod @@ -133,12 +133,15 @@ def get(cls, key): attrs = attrs or cache.get(cls.PREFIX_ID.format(key)) if not attrs: attrs = CITypeAttribute.get_by(type_id=key, to_dict=False) + if not attrs: ci_type = CIType.get_by(name=key, first=True, to_dict=False) if ci_type is not None: attrs = CITypeAttribute.get_by(type_id=ci_type.id, to_dict=False) + if attrs is not None: cls.set(key, attrs) + return attrs @classmethod @@ -155,13 +158,16 @@ def get2(cls, key): attrs = attrs or cache.get(cls.PREFIX_ID2.format(key)) if not attrs: attrs = CITypeAttribute.get_by(type_id=key, to_dict=False) + if not attrs: ci_type = CIType.get_by(name=key, first=True, to_dict=False) if ci_type is not None: attrs = CITypeAttribute.get_by(type_id=ci_type.id, to_dict=False) + if attrs is not None: attrs = [(i, AttributeCache.get(i.attr_id)) for i in attrs] cls.set2(key, attrs) + return attrs @classmethod @@ -201,13 +207,13 @@ class CITypeAttributeCache(object): @classmethod def get(cls, type_id, attr_id): - attr = cache.get(cls.PREFIX_ID.format(type_id, attr_id)) attr = attr or cache.get(cls.PREFIX_ID.format(type_id, attr_id)) - if not attr: - attr = CITypeAttribute.get_by(type_id=type_id, attr_id=attr_id, first=True, to_dict=False) - if attr is not None: - cls.set(type_id, attr_id, attr) + attr = attr or CITypeAttribute.get_by(type_id=type_id, attr_id=attr_id, first=True, to_dict=False) + + if attr is not None: + cls.set(type_id, attr_id, attr) + return attr @classmethod @@ -241,53 +247,72 @@ def reset(cls): result = {} for custom in customs: if custom['category'] == 0: - result[custom['id']] = cls.summary_counter(custom['type_id']) + res = cls.sum_counter(custom) elif custom['category'] == 1: - result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id']) - elif custom['category'] == 2: - result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level']) + res = cls.attribute_counter(custom) + else: + res = cls.relation_counter(custom.get('type_id'), + custom.get('level'), + custom.get('options', {}).get('filter', ''), + custom.get('options', {}).get('type_ids', '')) + + if res: + result[custom['id']] = res cls.set(result) return result @classmethod - def update(cls, custom): + def update(cls, custom, flush=True): result = cache.get(cls.KEY) or {} if not result: result = cls.reset() if custom['category'] == 0: - result[custom['id']] = cls.summary_counter(custom['type_id']) + res = cls.sum_counter(custom) elif custom['category'] == 1: - result[custom['id']] = cls.attribute_counter(custom['type_id'], custom['attr_id']) - elif custom['category'] == 2: - result[custom['id']] = cls.relation_counter(custom['type_id'], custom['level']) + res = cls.attribute_counter(custom) + else: + res = cls.relation_counter(custom.get('type_id'), + custom.get('level'), + custom.get('options', {}).get('filter', ''), + custom.get('options', {}).get('type_ids', '')) - cls.set(result) + if res and flush: + result[custom['id']] = res + cls.set(result) - @staticmethod - def summary_counter(type_id): - return db.session.query(CI.id).filter(CI.deleted.is_(False)).filter(CI.type_id == type_id).count() + return res @staticmethod - def relation_counter(type_id, level): - - uri = current_app.config.get('CMDB_API') + def relation_counter(type_id, level, other_filer, type_ids): + from api.lib.cmdb.search.ci_relation.search import Search as RelSearch + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + + query = "_type:{}".format(type_id) + s = search(query, count=1000000) + try: + type_names, _, _, _, _, _ = s.search() + except SearchError as e: + current_app.logger.error(e) + return - type_names = requests.get("{}/ci/s?q=_type:{}&count=10000".format(uri, type_id)).json().get('result') type_id_names = [(str(i.get('_id')), i.get(i.get('unique'))) for i in type_names] - url = "{}/ci_relations/statistics?root_ids={}&level={}".format( - uri, ','.join([i[0] for i in type_id_names]), level) - stats = requests.get(url).json() + s = RelSearch([i[0] for i in type_id_names], level, other_filer or '') + try: + stats = s.statistics(type_ids) + except SearchError as e: + current_app.logger.error(e) + return id2name = dict(type_id_names) type_ids = set() for i in (stats.get('detail') or []): for j in stats['detail'][i]: type_ids.add(j) - for type_id in type_ids: _type = CITypeCache.get(type_id) id2name[type_id] = _type and _type.alias @@ -307,9 +332,100 @@ def relation_counter(type_id, level): return result @staticmethod - def attribute_counter(type_id, attr_id): - uri = current_app.config.get('CMDB_API') - url = "{}/ci/s?q=_type:{}&fl={}&facet={}".format(uri, type_id, attr_id, attr_id) - res = requests.get(url).json() - if res.get('facet'): - return dict([i[:2] for i in list(res.get('facet').values())[0]]) + def attribute_counter(custom): + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + from api.lib.cmdb.utils import ValueTypeMap + + custom.setdefault('options', {}) + type_id = custom.get('type_id') + attr_id = custom.get('attr_id') + type_ids = custom['options'].get('type_ids') or (type_id and [type_id]) + attr_ids = list(map(str, custom['options'].get('attr_ids') or (attr_id and [attr_id]))) + try: + attr2value_type = [AttributeCache.get(i).value_type for i in attr_ids] + except AttributeError: + return + + other_filter = custom['options'].get('filter') + other_filter = "{}".format(other_filter) if other_filter else '' + + if custom['options'].get('ret') == 'cis': + query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter) + s = search(query, fl=attr_ids, ret_key='alias', count=100) + try: + cis, _, _, _, _, _ = s.search() + except SearchError as e: + current_app.logger.error(e) + return + + return cis + + result = dict() + # level = 1 + query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter) + s = search(query, fl=attr_ids, facet=[attr_ids[0]], count=1) + try: + _, _, _, _, _, facet = s.search() + except SearchError as e: + current_app.logger.error(e) + return + for i in (list(facet.values()) or [[]])[0]: + result[ValueTypeMap.serialize2[attr2value_type[0]](str(i[0]))] = i[1] + if len(attr_ids) == 1: + return result + + # level = 2 + for v in result: + query = "_type:({}),{},{}:{}".format(";".join(map(str, type_ids)), other_filter, attr_ids[0], v) + s = search(query, fl=attr_ids, facet=[attr_ids[1]], count=1) + try: + _, _, _, _, _, facet = s.search() + except SearchError as e: + current_app.logger.error(e) + return + result[v] = dict() + for i in (list(facet.values()) or [[]])[0]: + result[v][ValueTypeMap.serialize2[attr2value_type[1]](str(i[0]))] = i[1] + + if len(attr_ids) == 2: + return result + + # level = 3 + for v1 in result: + if not isinstance(result[v1], dict): + continue + for v2 in result[v1]: + query = "_type:({}),{},{}:{},{}:{}".format(";".join(map(str, type_ids)), other_filter, + attr_ids[0], v1, attr_ids[1], v2) + s = search(query, fl=attr_ids, facet=[attr_ids[2]], count=1) + try: + _, _, _, _, _, facet = s.search() + except SearchError as e: + current_app.logger.error(e) + return + result[v1][v2] = dict() + for i in (list(facet.values()) or [[]])[0]: + result[v1][v2][ValueTypeMap.serialize2[attr2value_type[2]](str(i[0]))] = i[1] + + return result + + @staticmethod + def sum_counter(custom): + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + + custom.setdefault('options', {}) + type_id = custom.get('type_id') + type_ids = custom['options'].get('type_ids') or (type_id and [type_id]) + other_filter = custom['options'].get('filter') or '' + + query = "_type:({}),{}".format(";".join(map(str, type_ids)), other_filter) + s = search(query, count=1) + try: + _, _, _, _, numfound, _ = s.search() + except SearchError as e: + current_app.logger.error(e) + return + + return numfound diff --git a/cmdb-api/api/lib/cmdb/ci.py b/cmdb-api/api/lib/cmdb/ci.py index e2c3cf31..4e8b9add 100644 --- a/cmdb-api/api/lib/cmdb/ci.py +++ b/cmdb-api/api/lib/cmdb/ci.py @@ -4,10 +4,11 @@ import copy import datetime import json +import threading from flask import abort from flask import current_app -from flask import g +from flask_login import current_user from werkzeug.exceptions import BadRequest from api.extensions import db @@ -24,31 +25,46 @@ from api.lib.cmdb.const import ConstraintEnum from api.lib.cmdb.const import ExistPolicy from api.lib.cmdb.const import OperateType +from api.lib.cmdb.const import PermEnum from api.lib.cmdb.const import REDIS_PREFIX_CI -from api.lib.cmdb.const import ResourceTypeEnum, PermEnum +from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.const import RetKey +from api.lib.cmdb.const import ValueTypeEnum from api.lib.cmdb.history import AttributeHistoryManger from api.lib.cmdb.history import CIRelationHistoryManager +from api.lib.cmdb.history import CITriggerHistoryManager from api.lib.cmdb.perms import CIFilterPermsCRUD from api.lib.cmdb.resp_format import ErrFormat from api.lib.cmdb.utils import TableMap from api.lib.cmdb.utils import ValueTypeMap from api.lib.cmdb.value import AttributeValueManager from api.lib.decorator import kwargs_required +from api.lib.notify import notify_send from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin from api.lib.perm.acl.acl import validate_permission +from api.lib.secrets.inner import InnerCrypt +from api.lib.secrets.vault import VaultClient from api.lib.utils import Lock from api.lib.utils import handle_arg_list +from api.lib.webhook import webhook_request +from api.models.cmdb import AttributeHistory +from api.models.cmdb import AutoDiscoveryCI from api.models.cmdb import CI from api.models.cmdb import CIRelation from api.models.cmdb import CITypeAttribute from api.models.cmdb import CITypeRelation +from api.models.cmdb import CITypeTrigger from api.tasks.cmdb import ci_cache from api.tasks.cmdb import ci_delete +from api.tasks.cmdb import ci_delete_trigger +from api.tasks.cmdb import ci_relation_add from api.tasks.cmdb import ci_relation_cache from api.tasks.cmdb import ci_relation_delete +PRIVILEGED_USERS = {"worker", "cmdb_agent", "agent"} +PASSWORD_DEFAULT_SHOW = "******" + class CIManager(object): """ manage CI interface @@ -64,11 +80,13 @@ def get_by_id(ci_id): @staticmethod def get_type_name(ci_id): ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(ci_id))) + return CITypeCache.get(ci.type_id).name @staticmethod def get_type(ci_id): ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(ci_id))) + return CITypeCache.get(ci.type_id) @staticmethod @@ -90,9 +108,7 @@ def get_ci_by_id(cls, ci_id, ret_key=RetKey.NAME, fields=None, need_children=Tru res = dict() - if need_children: - children = CIRelationManager.get_children(ci_id, ret_key=ret_key) # one floor - res.update(children) + need_children and res.update(CIRelationManager.get_children(ci_id, ret_key=ret_key)) # one floor ci_type = CITypeCache.get(ci.type_id) res["ci_type"] = ci_type.name @@ -159,14 +175,11 @@ def get_ci_by_id_from_db(cls, ci_id, ret_key=RetKey.NAME, fields=None, need_chil ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(ci_id))) - if valid: - cls.valid_ci_only_read(ci) + valid and cls.valid_ci_only_read(ci) res = dict() - if need_children: - children = CIRelationManager.get_children(ci_id, ret_key=ret_key) # one floor - res.update(children) + need_children and res.update(CIRelationManager.get_children(ci_id, ret_key=ret_key)) # one floor ci_type = CITypeCache.get(ci.type_id) res["ci_type"] = ci_type.name @@ -245,7 +258,7 @@ def _valid_unique_constraint(type_id, ci_dict, ci_id=None): for i in unique_constraints: attr_ids.extend(i.attr_ids) - attrs = [AttributeCache.get(i) for i in list(set(attr_ids))] + attrs = [AttributeCache.get(i) for i in set(attr_ids)] id2name = {i.id: i.name for i in attrs if i} not_existed_fields = list(set(id2name.values()) - set(ci_dict.keys())) if not_existed_fields and ci_id is not None: @@ -290,7 +303,7 @@ def add(cls, ci_type_name, _is_admin=False, **ci_dict): """ - + add ci :param ci_type_name: :param exist_policy: replace or reject or need :param _no_attribute_policy: ignore or reject @@ -305,9 +318,7 @@ def add(cls, ci_type_name, unique_key = AttributeCache.get(ci_type.unique_id) or abort( 400, ErrFormat.unique_value_not_found.format("unique_id={}".format(ci_type.unique_id))) - unique_value = ci_dict.get(unique_key.name) - unique_value = unique_value or ci_dict.get(unique_key.alias) - unique_value = unique_value or ci_dict.get(unique_key.id) + unique_value = ci_dict.get(unique_key.name) or ci_dict.get(unique_key.alias) or ci_dict.get(unique_key.id) unique_value = unique_value or abort(400, ErrFormat.unique_key_required.format(unique_key.name)) attrs = CITypeAttributesCache.get2(ci_type_name) @@ -316,7 +327,9 @@ def add(cls, ci_type_name, ci_attr2type_attr = {type_attr.attr_id: type_attr for type_attr, _ in attrs} ci = None - need_lock = g.user.username not in ("worker", "cmdb_agent", "agent") + record_id = None + password_dict = {} + need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS) with Lock(ci_type_name, need_lock=need_lock): existed = cls.ci_is_exist(unique_key, unique_value, ci_type.id) if existed is not None: @@ -330,10 +343,6 @@ def add(cls, ci_type_name, if exist_policy == ExistPolicy.NEED: return abort(404, ErrFormat.ci_not_found.format("{}={}".format(unique_key.name, unique_value))) - from api.lib.cmdb.const import L_CI - if L_CI and len(CI.get_by(type_id=ci_type.id)) > L_CI * 2: - return abort(400, ErrFormat.limit_ci.format(L_CI * 2)) - limit_attrs = cls._valid_ci_for_no_read(ci, ci_type) if not _is_admin else {} if existed is None: # set default @@ -348,14 +357,23 @@ def add(cls, ci_type_name, ci_dict.get(attr.name) is None and ci_dict.get(attr.alias) is None)): ci_dict[attr.name] = attr.default.get('default') - if type_attr.is_required and (attr.name not in ci_dict and attr.alias not in ci_dict): + if (type_attr.is_required and not attr.is_computed and + (attr.name not in ci_dict and attr.alias not in ci_dict)): return abort(400, ErrFormat.attribute_value_required.format(attr.name)) else: for type_attr, attr in attrs: if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT: ci_dict[attr.name] = now - computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None + computed_attrs = [] + for _, attr in attrs: + if attr.is_computed: + computed_attrs.append(attr.to_dict()) + elif attr.is_password: + if attr.name in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.name) + elif attr.alias in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.alias) value_manager = AttributeValueManager() @@ -364,13 +382,18 @@ def add(cls, ci_type_name, cls._valid_unique_constraint(ci_type.id, ci_dict, ci and ci.id) + ref_ci_dict = dict() for k in ci_dict: - if k not in ci_type_attrs_name and k not in ci_type_attrs_alias and \ - _no_attribute_policy == ExistPolicy.REJECT: + if k.startswith("$") and "." in k: + ref_ci_dict[k] = ci_dict[k] + continue + + if k not in ci_type_attrs_name and ( + k not in ci_type_attrs_alias and _no_attribute_policy == ExistPolicy.REJECT): return abort(400, ErrFormat.attribute_not_found.format(k)) - if limit_attrs and ci_type_attrs_name.get(k) not in limit_attrs and \ - ci_type_attrs_alias.get(k) not in limit_attrs: + if limit_attrs and ci_type_attrs_name.get(k) not in limit_attrs and ( + ci_type_attrs_alias.get(k) not in limit_attrs): return abort(403, ErrFormat.ci_filter_perm_attr_no_permission.format(k)) ci_dict = {k: v for k, v in ci_dict.items() if k in ci_type_attrs_name or k in ci_type_attrs_alias} @@ -378,16 +401,24 @@ def add(cls, ci_type_name, key2attr = value_manager.valid_attr_value(ci_dict, ci_type.id, ci and ci.id, ci_type_attrs_name, ci_type_attrs_alias, ci_attr2type_attr) + operate_type = OperateType.UPDATE if ci is not None else OperateType.ADD try: ci = ci or CI.create(type_id=ci_type.id, is_auto_discovery=is_auto_discovery) - record_id = value_manager.create_or_update_attr_value2(ci, ci_dict, key2attr) + record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr) except BadRequest as e: if existed is None: cls.delete(ci.id) raise e + if password_dict: + for attr_id in password_dict: + record_id = cls.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci_type.id) + if record_id: # has change - ci_cache.apply_async([ci.id], queue=CMDB_QUEUE) + ci_cache.apply_async(args=(ci.id, operate_type, record_id), queue=CMDB_QUEUE) + + if ref_ci_dict: # add relations + ci_relation_add.apply_async(args=(ref_ci_dict, ci.id, current_user.uid), queue=CMDB_QUEUE) return ci.id @@ -402,7 +433,16 @@ def update(self, ci_id, _is_admin=False, **ci_dict): if attr.default and attr.default.get('default') == AttributeDefaultValueEnum.UPDATED_AT: ci_dict[attr.name] = now - computed_attrs = [attr.to_dict() for _, attr in attrs if attr.is_computed] or None + password_dict = dict() + computed_attrs = list() + for _, attr in attrs: + if attr.is_computed: + computed_attrs.append(attr.to_dict()) + elif attr.is_password: + if attr.name in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.name) + elif attr.alias in ci_dict: + password_dict[attr.id] = ci_dict.pop(attr.alias) value_manager = AttributeValueManager() @@ -411,7 +451,8 @@ def update(self, ci_id, _is_admin=False, **ci_dict): limit_attrs = self._valid_ci_for_no_read(ci) if not _is_admin else {} - need_lock = g.user.username not in ("worker", "cmdb_agent", "agent") + record_id = None + need_lock = current_user.username not in current_app.config.get('PRIVILEGED_USERS', PRIVILEGED_USERS) with Lock(ci.ci_type.name, need_lock=need_lock): self._valid_unique_constraint(ci.type_id, ci_dict, ci_id) @@ -424,20 +465,29 @@ def update(self, ci_id, _is_admin=False, **ci_dict): return abort(403, ErrFormat.ci_filter_perm_attr_no_permission.format(k)) try: - record_id = value_manager.create_or_update_attr_value2(ci, ci_dict, key2attr) + record_id = value_manager.create_or_update_attr_value(ci, ci_dict, key2attr) except BadRequest as e: raise e + if password_dict: + for attr_id in password_dict: + record_id = self.save_password(ci.id, attr_id, password_dict[attr_id], record_id, ci.type_id) + if record_id: # has change - ci_cache.apply_async([ci_id], queue=CMDB_QUEUE) + ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE) + + ref_ci_dict = {k: v for k, v in ci_dict.items() if k.startswith("$") and "." in k} + if ref_ci_dict: + ci_relation_add.apply_async(args=(ref_ci_dict, ci.id), queue=CMDB_QUEUE) @staticmethod def update_unique_value(ci_id, unique_name, unique_value): ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(ci_id))) - AttributeValueManager().create_or_update_attr_value(unique_name, unique_value, ci) + key2attr = {unique_name: AttributeCache.get(unique_name)} + record_id = AttributeValueManager().create_or_update_attr_value(ci, {unique_name: unique_value}, key2attr) - ci_cache.apply_async([ci_id], queue=CMDB_QUEUE) + ci_cache.apply_async(args=(ci_id, OperateType.UPDATE, record_id), queue=CMDB_QUEUE) @classmethod def delete(cls, ci_id): @@ -448,26 +498,44 @@ def delete(cls, ci_id): ci_dict = cls.get_cis_by_ids([ci_id]) ci_dict = ci_dict and ci_dict[0] + if ci_dict: + triggers = CITriggerManager.get(ci_dict['_type']) + for trigger in triggers: + option = trigger['option'] + if not option.get('enable') or option.get('action') != OperateType.DELETE: + continue + + if option.get('filter') and not CITriggerManager.ci_filter(ci_dict.get('_id'), option['filter']): + continue + + ci_delete_trigger.apply_async(args=(trigger, OperateType.DELETE, ci_dict), queue=CMDB_QUEUE) + attrs = CITypeAttribute.get_by(type_id=ci.type_id, to_dict=False) attr_names = set([AttributeCache.get(attr.attr_id).name for attr in attrs]) for attr_name in attr_names: value_table = TableMap(attr_name=attr_name).table for item in value_table.get_by(ci_id=ci_id, to_dict=False): - item.delete() + item.delete(commit=False) for item in CIRelation.get_by(first_ci_id=ci_id, to_dict=False): ci_relation_delete.apply_async(args=(item.first_ci_id, item.second_ci_id), queue=CMDB_QUEUE) - item.delete() + item.delete(commit=False) for item in CIRelation.get_by(second_ci_id=ci_id, to_dict=False): ci_relation_delete.apply_async(args=(item.first_ci_id, item.second_ci_id), queue=CMDB_QUEUE) - item.delete() + item.delete(commit=False) - ci.delete() # TODO: soft delete + ad_ci = AutoDiscoveryCI.get_by(ci_id=ci_id, to_dict=False, first=True) + ad_ci and ad_ci.update(is_accept=False, accept_by=None, accept_time=None, filter_none=False, commit=False) + + ci.delete(commit=False) # TODO: soft delete + + db.session.commit() - AttributeHistoryManger.add(None, ci_id, [(None, OperateType.DELETE, ci_dict, None)], ci.type_id) + if ci_dict: + AttributeHistoryManger.add(None, ci_id, [(None, OperateType.DELETE, ci_dict, None)], ci.type_id) - ci_delete.apply_async([ci.id], queue=CMDB_QUEUE) + ci_delete.apply_async(args=(ci_id,), queue=CMDB_QUEUE) return ci_id @@ -478,11 +546,8 @@ def add_heartbeat(ci_type, unique_value): unique_key = AttributeCache.get(ci_type.unique_id) value_table = TableMap(attr=unique_key).table - v = value_table.get_by(attr_id=unique_key.id, - value=unique_value, - to_dict=False, - first=True) \ - or abort(404, ErrFormat.not_found) + v = (value_table.get_by(attr_id=unique_key.id, value=unique_value, to_dict=False, first=True) or + abort(404, ErrFormat.not_found)) ci = CI.get_by_id(v.ci_id) or abort(404, ErrFormat.ci_not_found.format("id={}".format(v.ci_id))) @@ -528,6 +593,7 @@ def get_heartbeat(cls, **kwargs): result = [(i.get("hostname"), i.get("private_ip")[0], i.get("ci_type"), heartbeat_dict.get(i.get("_id"))) for i in res if i.get("private_ip")] + return numfound, result @staticmethod @@ -569,10 +635,13 @@ def _get_cis_from_db(ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None _fields = list() for field in fields: attr = AttributeCache.get(field) - if attr is not None: + if attr is not None and not attr.is_password: _fields.append(str(attr.id)) filter_fields_sql = "WHERE A.attr_id in ({0})".format(",".join(_fields)) + ci2pos = {int(_id): _pos for _pos, _id in enumerate(ci_ids)} + res = [None] * len(ci_ids) + ci_ids = ",".join(map(str, ci_ids)) if value_tables is None: value_tables = ValueTypeMap.table_name.values() @@ -583,11 +652,10 @@ def _get_cis_from_db(ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None # current_app.logger.debug(query_sql) cis = db.session.execute(query_sql).fetchall() ci_set = set() - res = list() ci_dict = dict() unique_id2obj = dict() excludes = excludes and set(excludes) - for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list in cis: + for ci_id, type_id, attr_id, attr_name, attr_alias, value, value_type, is_list, is_password in cis: if not fields and excludes and (attr_name in excludes or attr_alias in excludes): continue @@ -603,7 +671,7 @@ def _get_cis_from_db(ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None ci_dict["unique"] = unique_id2obj[ci_type.unique_id] and unique_id2obj[ci_type.unique_id].name ci_dict["unique_alias"] = unique_id2obj[ci_type.unique_id] and unique_id2obj[ci_type.unique_id].alias ci_set.add(ci_id) - res.append(ci_dict) + res[ci2pos[ci_id]] = ci_dict if ret_key == RetKey.NAME: attr_key = attr_name @@ -614,11 +682,14 @@ def _get_cis_from_db(ci_ids, ret_key=RetKey.NAME, fields=None, value_tables=None else: return abort(400, ErrFormat.argument_invalid.format("ret_key")) - value = ValueTypeMap.serialize2[value_type](value) - if is_list: - ci_dict.setdefault(attr_key, []).append(value) + if is_password and value: + ci_dict[attr_key] = PASSWORD_DEFAULT_SHOW else: - ci_dict[attr_key] = value + value = ValueTypeMap.serialize2[value_type](value) + if is_list: + ci_dict.setdefault(attr_key, []).append(value) + else: + ci_dict[attr_key] = value return res @@ -647,8 +718,87 @@ def get_cis_by_ids(cls, ci_ids, ret_key=RetKey.NAME, return res current_app.logger.warning("cache not hit...............") + return cls._get_cis_from_db(ci_ids, ret_key, fields, value_tables, excludes=excludes) + @classmethod + def save_password(cls, ci_id, attr_id, value, record_id, type_id): + changed = None + encrypt_value = None + value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD] + if current_app.config.get('SECRETS_ENGINE') == 'inner': + if value: + encrypt_value, status = InnerCrypt().encrypt(value) + if not status: + current_app.logger.error('save password failed: {}'.format(encrypt_value)) + return abort(400, ErrFormat.password_save_failed.format(encrypt_value)) + else: + encrypt_value = PASSWORD_DEFAULT_SHOW + + existed = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False) + if existed is None: + if value: + value_table.create(ci_id=ci_id, attr_id=attr_id, value=encrypt_value) + changed = [(ci_id, attr_id, OperateType.ADD, '', PASSWORD_DEFAULT_SHOW, type_id)] + elif existed.value != encrypt_value: + if value: + existed.update(ci_id=ci_id, attr_id=attr_id, value=encrypt_value) + changed = [(ci_id, attr_id, OperateType.UPDATE, PASSWORD_DEFAULT_SHOW, PASSWORD_DEFAULT_SHOW, type_id)] + else: + existed.delete() + changed = [(ci_id, attr_id, OperateType.DELETE, PASSWORD_DEFAULT_SHOW, '', type_id)] + + if current_app.config.get('SECRETS_ENGINE') == 'vault': + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + if value: + try: + vault.update("/{}/{}".format(ci_id, attr_id), dict(v=value)) + except Exception as e: + current_app.logger.error('save password to vault failed: {}'.format(e)) + return abort(400, ErrFormat.password_save_failed.format('write vault failed')) + else: + try: + vault.delete("/{}/{}".format(ci_id, attr_id)) + except Exception as e: + current_app.logger.warning('delete password to vault failed: {}'.format(e)) + + if changed is not None: + return AttributeValueManager.write_change2(changed, record_id) + + @classmethod + def load_password(cls, ci_id, attr_id): + ci = CI.get_by_id(ci_id) or abort(404, ErrFormat.ci_not_found.format(ci_id)) + + limit_attrs = cls._valid_ci_for_no_read(ci, ci.ci_type) + if limit_attrs: + attr = AttributeCache.get(attr_id) + if attr and attr.name not in limit_attrs: + return abort(403, ErrFormat.no_permission2) + + if current_app.config.get('SECRETS_ENGINE', 'inner') == 'inner': + value_table = ValueTypeMap.table[ValueTypeEnum.PASSWORD] + v = value_table.get_by(ci_id=ci_id, attr_id=attr_id, first=True, to_dict=False) + + v = v and v.value + if not v: + return + + decrypt_value, status = InnerCrypt().decrypt(v) + if not status: + current_app.logger.error('load password failed: {}'.format(decrypt_value)) + return abort(400, ErrFormat.password_load_failed.format(decrypt_value)) + + return decrypt_value + + elif current_app.config.get('SECRETS_ENGINE') == 'vault': + vault = VaultClient(current_app.config.get('VAULT_URL'), current_app.config.get('VAULT_TOKEN')) + data, status = vault.read("/{}/{}".format(ci_id, attr_id)) + if not status: + current_app.logger.error('read password from vault failed: {}'.format(data)) + return abort(400, ErrFormat.password_load_failed.format(data)) + + return data.get('v') + class CIRelationManager(object): """ @@ -672,6 +822,7 @@ def get_children(cls, ci_id, ret_key=RetKey.NAME): ci_type = CITypeCache.get(type_id) children = CIManager.get_cis_by_ids(list(map(str, ci_type2ci_ids[type_id])), ret_key=ret_key) res[ci_type.name] = children + return res @staticmethod @@ -743,17 +894,28 @@ def get_ancestor_ids(cls, ci_ids, level=1): return ci_ids @staticmethod - def _check_constraint(first_ci_id, second_ci_id, type_relation): + def _check_constraint(first_ci_id, first_type_id, second_ci_id, second_type_id, type_relation): + db.session.remove() if type_relation.constraint == ConstraintEnum.Many2Many: return - first_existed = CIRelation.get_by(first_ci_id=first_ci_id, relation_type_id=type_relation.relation_type_id) - second_existed = CIRelation.get_by(second_ci_id=second_ci_id, relation_type_id=type_relation.relation_type_id) - if type_relation.constraint == ConstraintEnum.One2One and (first_existed or second_existed): - return abort(400, ErrFormat.relation_constraint.format("1对1")) + first_existed = CIRelation.get_by(first_ci_id=first_ci_id, + relation_type_id=type_relation.relation_type_id, to_dict=False) + second_existed = CIRelation.get_by(second_ci_id=second_ci_id, + relation_type_id=type_relation.relation_type_id, to_dict=False) + if type_relation.constraint == ConstraintEnum.One2One: + for i in first_existed: + if i.second_ci.type_id == second_type_id: + return abort(400, ErrFormat.relation_constraint.format("1-1")) - if type_relation.constraint == ConstraintEnum.One2Many and second_existed: - return abort(400, ErrFormat.relation_constraint.format("1对多")) + for i in second_existed: + if i.first_ci.type_id == first_type_id: + return abort(400, ErrFormat.relation_constraint.format("1-1")) + + if type_relation.constraint == ConstraintEnum.One2Many: + for i in second_existed: + if i.first_ci.type_id == first_type_id: + return abort(400, ErrFormat.relation_constraint.format("1-N")) @classmethod def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None): @@ -792,15 +954,17 @@ def add(cls, first_ci_id, second_ci_id, more=None, relation_type_id=None): else: type_relation = CITypeRelation.get_by_id(relation_type_id) - cls._check_constraint(first_ci_id, second_ci_id, type_relation) + with Lock("ci_relation_add_{}_{}".format(first_ci.type_id, second_ci.type_id), need_lock=True): + + cls._check_constraint(first_ci_id, first_ci.type_id, second_ci_id, second_ci.type_id, type_relation) - existed = CIRelation.create(first_ci_id=first_ci_id, - second_ci_id=second_ci_id, - relation_type_id=relation_type_id) + existed = CIRelation.create(first_ci_id=first_ci_id, + second_ci_id=second_ci_id, + relation_type_id=relation_type_id) - CIRelationHistoryManager().add(existed, OperateType.ADD) + CIRelationHistoryManager().add(existed, OperateType.ADD) - ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE) + ci_relation_cache.apply_async(args=(first_ci_id, second_ci_id), queue=CMDB_QUEUE) if more is not None: existed.upadte(more=more) @@ -848,12 +1012,12 @@ def batch_update(cls, ci_ids, parents, children): :param children: :return: """ - if parents is not None and isinstance(parents, list): + if isinstance(parents, list): for parent_id in parents: for ci_id in ci_ids: cls.add(parent_id, ci_id) - if children is not None and isinstance(children, list): + if isinstance(children, list): for child_id in children: for ci_id in ci_ids: cls.add(ci_id, child_id) @@ -867,7 +1031,184 @@ def batch_delete(cls, ci_ids, parents): :return: """ - if parents is not None and isinstance(parents, list): + if isinstance(parents, list): for parent_id in parents: for ci_id in ci_ids: cls.delete_2(parent_id, ci_id) + + +class CITriggerManager(object): + @staticmethod + def get(type_id): + db.session.remove() + return CITypeTrigger.get_by(type_id=type_id, to_dict=True) + + @staticmethod + def _update_old_attr_value(record_id, ci_dict): + attr_history = AttributeHistory.get_by(record_id=record_id, to_dict=False) + attr_dict = dict() + for attr_h in attr_history: + attr_dict['old_{}'.format(AttributeCache.get(attr_h.attr_id).name)] = attr_h.old + + ci_dict.update({'old_{}'.format(k): ci_dict[k] for k in ci_dict}) + + ci_dict.update(attr_dict) + + @classmethod + def _exec_webhook(cls, operate_type, webhook, ci_dict, trigger_id, trigger_name, record_id, ci_id=None, app=None): + app = app or current_app + + with app.app_context(): + if operate_type == OperateType.UPDATE: + cls._update_old_attr_value(record_id, ci_dict) + + if ci_id is not None: + ci_dict = CIManager().get_ci_by_id_from_db(ci_id, need_children=False, use_master=False) + + try: + response = webhook_request(webhook, ci_dict).text + is_ok = True + except Exception as e: + current_app.logger.warning("exec webhook failed: {}".format(e)) + response = e + is_ok = False + + CITriggerHistoryManager.add(operate_type, + record_id, + ci_dict.get('_id'), + trigger_id, + trigger_name, + is_ok=is_ok, + webhook=response) + + return is_ok + + @classmethod + def _exec_notify(cls, operate_type, notify, ci_dict, trigger_id, trigger_name, record_id, ci_id=None, app=None): + app = app or current_app + + with app.app_context(): + + if ci_id is not None: + ci_dict = CIManager().get_ci_by_id_from_db(ci_id, need_children=False, use_master=False) + + if operate_type == OperateType.UPDATE: + cls._update_old_attr_value(record_id, ci_dict) + + is_ok = True + response = '' + for method in (notify.get('method') or []): + try: + res = notify_send(notify.get('subject'), notify.get('body'), [method], + notify.get('tos'), ci_dict) + response = "{}\n{}".format(response, res) + except Exception as e: + current_app.logger.warning("send notify failed: {}".format(e)) + response = "{}\n{}".format(response, e) + is_ok = False + + CITriggerHistoryManager.add(operate_type, + record_id, + ci_dict.get('_id'), + trigger_id, + trigger_name, + is_ok=is_ok, + notify=response.strip()) + + return is_ok + + @staticmethod + def ci_filter(ci_id, other_filter): + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + + query = "{},_id:{}".format(other_filter, ci_id) + + try: + _, _, _, _, numfound, _ = search(query).search() + return numfound + except SearchError as e: + current_app.logger.warning("ci search failed: {}".format(e)) + + @classmethod + def fire(cls, operate_type, ci_dict, record_id): + type_id = ci_dict.get('_type') + triggers = cls.get(type_id) or [] + + for trigger in triggers: + option = trigger['option'] + if not option.get('enable'): + continue + + if option.get('filter') and not cls.ci_filter(ci_dict.get('_id'), option['filter']): + continue + + if option.get('attr_ids') and isinstance(option['attr_ids'], list): + if not (set(option['attr_ids']) & + set([i.attr_id for i in AttributeHistory.get_by(record_id=record_id, to_dict=False)])): + continue + + if option.get('action') == operate_type: + cls.fire_by_trigger(trigger, operate_type, ci_dict, record_id) + + @classmethod + def fire_by_trigger(cls, trigger, operate_type, ci_dict, record_id=None): + option = trigger['option'] + + if option.get('webhooks'): + cls._exec_webhook(operate_type, option['webhooks'], ci_dict, trigger['id'], + option.get('name'), record_id) + + elif option.get('notifies'): + cls._exec_notify(operate_type, option['notifies'], ci_dict, trigger['id'], + option.get('name'), record_id) + + @classmethod + def waiting_cis(cls, trigger): + now = datetime.datetime.today() + + config = trigger.option.get('notifies') or {} + + delta_time = datetime.timedelta(days=(config.get('before_days', 0) or 0)) + + attr = AttributeCache.get(trigger.attr_id) + + value_table = TableMap(attr=attr).table + + values = value_table.get_by(attr_id=attr.id, to_dict=False) + + result = [] + for v in values: + if (isinstance(v.value, (datetime.date, datetime.datetime)) and + (v.value - delta_time).strftime('%Y%m%d') == now.strftime("%Y%m%d")): + + if trigger.option.get('filter') and not cls.ci_filter(v.ci_id, trigger.option['filter']): + continue + + result.append(v) + + return result + + @classmethod + def trigger_notify(cls, trigger, ci): + """ + only for date attribute + :param trigger: + :param ci: + :return: + """ + if (trigger.option.get('notifies', {}).get('notify_at') == datetime.datetime.now().strftime("%H:%M") or + not trigger.option.get('notifies', {}).get('notify_at')): + + if trigger.option.get('webhooks'): + threading.Thread(target=cls._exec_webhook, args=( + None, trigger.option['webhooks'], None, trigger.id, trigger.option.get('name'), None, ci.ci_id, + current_app._get_current_object())).start() + elif trigger.option.get('notifies'): + threading.Thread(target=cls._exec_notify, args=( + None, trigger.option['notifies'], None, trigger.id, trigger.option.get('name'), None, ci.ci_id, + current_app._get_current_object())).start() + + return True + + return False diff --git a/cmdb-api/api/lib/cmdb/ci_type.py b/cmdb-api/api/lib/cmdb/ci_type.py index a88a16b8..4adefad1 100644 --- a/cmdb-api/api/lib/cmdb/ci_type.py +++ b/cmdb-api/api/lib/cmdb/ci_type.py @@ -1,11 +1,12 @@ -# -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- import copy -import datetime +import toposort from flask import abort from flask import current_app -from flask import g +from flask_login import current_user +from toposort import toposort_flatten from api.extensions import db from api.lib.cmdb.attribute import AttributeManager @@ -16,18 +17,22 @@ from api.lib.cmdb.const import CITypeOperateType from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import ConstraintEnum -from api.lib.cmdb.const import PermEnum, ResourceTypeEnum, RoleEnum +from api.lib.cmdb.const import PermEnum +from api.lib.cmdb.const import ResourceTypeEnum +from api.lib.cmdb.const import RoleEnum from api.lib.cmdb.const import ValueTypeEnum from api.lib.cmdb.history import CITypeHistoryManager from api.lib.cmdb.relation_type import RelationTypeManager from api.lib.cmdb.resp_format import ErrFormat -from api.lib.cmdb.utils import TableMap from api.lib.cmdb.value import AttributeValueManager from api.lib.decorator import kwargs_required from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin from api.models.cmdb import Attribute +from api.models.cmdb import AutoDiscoveryCI +from api.models.cmdb import AutoDiscoveryCIType from api.models.cmdb import CI +from api.models.cmdb import CIFilterPerms from api.models.cmdb import CIType from api.models.cmdb import CITypeAttribute from api.models.cmdb import CITypeAttributeGroup @@ -37,6 +42,9 @@ from api.models.cmdb import CITypeRelation from api.models.cmdb import CITypeTrigger from api.models.cmdb import CITypeUniqueConstraint +from api.models.cmdb import CustomDashboard +from api.models.cmdb import PreferenceRelationView +from api.models.cmdb import PreferenceSearchOption from api.models.cmdb import PreferenceShowAttributes from api.models.cmdb import PreferenceTreeView from api.models.cmdb import RelationType @@ -54,6 +62,7 @@ def __init__(self): @staticmethod def get_name_by_id(type_id): ci_type = CITypeCache.get(type_id) + return ci_type and ci_type.name @staticmethod @@ -65,7 +74,7 @@ def check_is_existed(key): @staticmethod def get_ci_types(type_name=None): resources = None - if current_app.config.get('USE_ACL') and not is_app_admin(): + if current_app.config.get('USE_ACL') and not is_app_admin('cmdb'): resources = set([i.get('name') for i in ACLManager().get_resources("CIType")]) ci_types = CIType.get_by() if type_name is None else CIType.get_by_like(name=type_name) @@ -104,11 +113,8 @@ def _validate_unique(type_id=None, name=None, alias=None): @classmethod @kwargs_required("name") def add(cls, **kwargs): - from api.lib.cmdb.const import L_TYPE - if L_TYPE and len(CIType.get_by()) > L_TYPE * 2: - return abort(400, ErrFormat.limit_ci_type.format(L_TYPE * 2)) - unique_key = kwargs.pop("unique_key", None) + unique_key = kwargs.pop("unique_key", None) or kwargs.pop("unique_id", None) unique_key = AttributeCache.get(unique_key) or abort(404, ErrFormat.unique_key_not_define) kwargs["alias"] = kwargs["name"] if not kwargs.get("alias") else kwargs["alias"] @@ -117,7 +123,7 @@ def add(cls, **kwargs): cls._validate_unique(alias=kwargs['alias']) kwargs["unique_id"] = unique_key.id - kwargs['uid'] = g.user.uid + kwargs['uid'] = current_user.uid ci_type = CIType.create(**kwargs) CITypeAttributeManager.add(ci_type.id, [unique_key.id], is_required=True) @@ -131,7 +137,7 @@ def add(cls, **kwargs): ResourceTypeEnum.CI, permissions=[PermEnum.READ]) ACLManager().grant_resource_to_role(ci_type.name, - g.user.username, + current_user.username, ResourceTypeEnum.CI) CITypeHistoryManager.add(CITypeOperateType.ADD, ci_type.id, change=ci_type.to_dict()) @@ -178,32 +184,41 @@ def update(cls, type_id, **kwargs): def set_enabled(cls, type_id, enabled=True): ci_type = cls.check_is_existed(type_id) ci_type.update(enabled=enabled) + return type_id @classmethod def delete(cls, type_id): ci_type = cls.check_is_existed(type_id) - if ci_type.uid and ci_type.uid != g.user.uid: + if ci_type.uid and ci_type.uid != current_user.uid: return abort(403, ErrFormat.only_owner_can_delete) if CI.get_by(type_id=type_id, first=True, to_dict=False) is not None: return abort(400, ErrFormat.ci_exists_and_cannot_delete_type) + relation_views = PreferenceRelationView.get_by(to_dict=False) + for rv in relation_views: + for item in (rv.cr_ids or []): + if item.get('parent_id') == type_id or item.get('child_id') == type_id: + return abort(400, ErrFormat.ci_relation_view_exists_and_cannot_delete_type.format(rv.name)) + for item in CITypeRelation.get_by(parent_id=type_id, to_dict=False): - item.soft_delete() + item.soft_delete(commit=False) for item in CITypeRelation.get_by(child_id=type_id, to_dict=False): - item.soft_delete() + item.soft_delete(commit=False) - for item in PreferenceTreeView.get_by(type_id=type_id, to_dict=False): - item.soft_delete() + for table in [PreferenceTreeView, PreferenceShowAttributes, PreferenceSearchOption, CustomDashboard, + CITypeGroupItem, CITypeAttributeGroup, CITypeAttribute, CITypeUniqueConstraint, CITypeTrigger, + AutoDiscoveryCIType, CIFilterPerms]: + for item in table.get_by(type_id=type_id, to_dict=False): + item.soft_delete(commit=False) - for item in PreferenceShowAttributes.get_by(type_id=type_id, to_dict=False): - item.soft_delete() + for item in AutoDiscoveryCI.get_by(type_id=type_id, to_dict=False): + item.delete(commit=False) - for item in CITypeGroupItem.get_by(type_id=type_id, to_dict=False): - item.soft_delete() + db.session.commit() ci_type.soft_delete() @@ -254,16 +269,17 @@ def get(need_other=None, config_required=True): @staticmethod def add(name): CITypeGroup.get_by(name=name, first=True) and abort(400, ErrFormat.ci_type_group_exists.format(name)) + return CITypeGroup.create(name=name) @staticmethod def update(gid, name, type_ids): """ update part - :param gid: - :param name: - :param type_ids: - :return: + :param gid: + :param name: + :param type_ids: + :return: """ existed = CITypeGroup.get_by_id(gid) or abort( 404, ErrFormat.ci_type_group_not_found.format("id={}".format(gid))) @@ -320,28 +336,63 @@ class CITypeAttributeManager(object): def __init__(self): pass + @staticmethod + def get_attr_name(ci_type_name, key): + ci_type = CITypeCache.get(ci_type_name) + if ci_type is None: + return + + for i in CITypeAttributesCache.get(ci_type.id): + attr = AttributeCache.get(i.attr_id) + if attr and (attr.name == key or attr.alias == key): + return attr.name + @staticmethod def get_attr_names_by_type_id(type_id): return [AttributeCache.get(attr.attr_id).name for attr in CITypeAttributesCache.get(type_id)] @staticmethod - def get_attributes_by_type_id(type_id, choice_web_hook_parse=True): + def get_attributes_by_type_id(type_id, choice_web_hook_parse=True, choice_other_parse=True): has_config_perm = ACLManager('cmdb').has_permission( CITypeManager.get_name_by_id(type_id), ResourceTypeEnum.CI, PermEnum.CONFIG) attrs = CITypeAttributesCache.get(type_id) result = list() for attr in sorted(attrs, key=lambda x: (x.order, x.id)): - attr_dict = AttributeManager().get_attribute(attr.attr_id, choice_web_hook_parse) + attr_dict = AttributeManager().get_attribute(attr.attr_id, choice_web_hook_parse, choice_other_parse) attr_dict["is_required"] = attr.is_required attr_dict["order"] = attr.order attr_dict["default_show"] = attr.default_show if not has_config_perm: attr_dict.pop('choice_web_hook', None) + attr_dict.pop('choice_other', None) result.append(attr_dict) + return result + @staticmethod + def get_common_attributes(type_ids): + has_config_perm = False + for type_id in type_ids: + has_config_perm |= ACLManager('cmdb').has_permission( + CITypeManager.get_name_by_id(type_id), ResourceTypeEnum.CI, PermEnum.CONFIG) + + result = CITypeAttribute.get_by(__func_in___key_type_id=list(map(int, type_ids)), to_dict=False) + attr2types = {} + for i in result: + attr2types.setdefault(i.attr_id, []).append(i.type_id) + + attrs = [] + for attr_id in attr2types: + if len(attr2types[attr_id]) == len(type_ids): + attr = AttributeManager().get_attribute_by_id(attr_id) + if not has_config_perm: + attr.pop('choice_web_hook', None) + attrs.append(attr) + + return attrs + @staticmethod def _check(type_id, attr_ids): ci_type = CITypeManager.check_is_existed(type_id) @@ -358,10 +409,10 @@ def _check(type_id, attr_ids): def add(cls, type_id, attr_ids=None, **kwargs): """ add attributes to CIType - :param type_id: + :param type_id: :param attr_ids: list - :param kwargs: - :return: + :param kwargs: + :return: """ attr_ids = list(set(attr_ids)) @@ -388,9 +439,9 @@ def add(cls, type_id, attr_ids=None, **kwargs): def update(cls, type_id, attributes): """ update attributes to CIType - :param type_id: + :param type_id: :param attributes: list - :return: + :return: """ cls._check(type_id, [i.get('attr_id') for i in attributes]) @@ -418,9 +469,9 @@ def update(cls, type_id, attributes): def delete(cls, type_id, attr_ids=None): """ delete attributes from CIType - :param type_id: + :param type_id: :param attr_ids: list - :return: + :return: """ from api.tasks.cmdb import ci_cache @@ -449,7 +500,7 @@ def delete(cls, type_id, attr_ids=None): for ci in CI.get_by(type_id=type_id, to_dict=False): AttributeValueManager.delete_attr_value(attr_id, ci.id) - ci_cache.apply_async([ci.id], queue=CMDB_QUEUE) + ci_cache.apply_async(args=(ci.id, None, None), queue=CMDB_QUEUE) CITypeAttributeCache.clean(type_id, attr_id) @@ -482,7 +533,7 @@ def transfer(cls, type_id, _from, _to): CITypeAttributesCache.clean(type_id) from api.tasks.cmdb import ci_type_attribute_order_rebuild - ci_type_attribute_order_rebuild.apply_async(args=(type_id,), queue=CMDB_QUEUE) + ci_type_attribute_order_rebuild.apply_async(args=(type_id, current_user.uid), queue=CMDB_QUEUE) class CITypeRelationManager(object): @@ -527,6 +578,7 @@ def _wrap_relation_type_dict(type_id, relation_inst): ci_type_dict["attributes"] = CITypeAttributeManager.get_attributes_by_type_id(ci_type_dict["id"]) ci_type_dict["relation_type"] = relation_inst.relation_type.name ci_type_dict["constraint"] = relation_inst.constraint + return ci_type_dict @classmethod @@ -535,6 +587,23 @@ def get_children(cls, parent_id): return [cls._wrap_relation_type_dict(child.child_id, child) for child in children] + @classmethod + def recursive_level2children(cls, parent_id): + result = dict() + + def get_children(_id, level): + children = CITypeRelation.get_by(parent_id=_id, to_dict=False) + if children: + result.setdefault(level + 1, []).extend([i.child.to_dict() for i in children]) + + for i in children: + if i.child_id != _id: + get_children(i.child_id, level + 1) + + get_children(parent_id, 0) + + return result + @classmethod def get_parents(cls, child_id): parents = CITypeRelation.get_by(child_id=child_id, to_dict=False) @@ -557,6 +626,17 @@ def add(cls, parent, child, relation_type_id, constraint=ConstraintEnum.One2Many p = CITypeManager.check_is_existed(parent) c = CITypeManager.check_is_existed(child) + rels = {} + for i in CITypeRelation.get_by(to_dict=False): + rels.setdefault(i.child_id, set()).add(i.parent_id) + rels.setdefault(c.id, set()).add(p.id) + + try: + toposort_flatten(rels) + except toposort.CircularDependencyError as e: + current_app.logger.warning(str(e)) + return abort(400, ErrFormat.circular_dependency_error) + existed = cls._get(p.id, c.id) if existed is not None: existed.update(relation_type_id=relation_type_id, @@ -575,7 +655,7 @@ def add(cls, parent, child, relation_type_id, constraint=ConstraintEnum.One2Many ResourceTypeEnum.CI_TYPE_RELATION, permissions=[PermEnum.READ]) ACLManager().grant_resource_to_role(resource_name, - g.user.username, + current_user.username, ResourceTypeEnum.CI_TYPE_RELATION) CITypeHistoryManager.add(CITypeOperateType.ADD_RELATION, p.id, @@ -585,8 +665,8 @@ def add(cls, parent, child, relation_type_id, constraint=ConstraintEnum.One2Many @classmethod def delete(cls, _id): - ctr = CITypeRelation.get_by_id(_id) or \ - abort(404, ErrFormat.ci_type_relation_not_found.format("id={}".format(_id))) + ctr = (CITypeRelation.get_by_id(_id) or + abort(404, ErrFormat.ci_type_relation_not_found.format("id={}".format(_id)))) ctr.soft_delete() CITypeHistoryManager.add(CITypeOperateType.DELETE_RELATION, ctr.parent_id, @@ -640,6 +720,7 @@ def create_or_update(type_id, name, attr_order, group_order=0, is_update=False): :param name: :param group_order: group order :param attr_order: + :param is_update: :return: """ existed = CITypeAttributeGroup.get_by(type_id=type_id, name=name, first=True, to_dict=False) @@ -680,8 +761,8 @@ def update(cls, group_id, name, attr_order, group_order=0): @staticmethod def delete(group_id): - group = CITypeAttributeGroup.get_by_id(group_id) \ - or abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(group_id))) + group = (CITypeAttributeGroup.get_by_id(group_id) or + abort(404, ErrFormat.ci_type_attribute_group_not_found.format("id={}".format(group_id)))) group.soft_delete() items = CITypeAttributeGroupItem.get_by(group_id=group_id, to_dict=False) @@ -777,7 +858,7 @@ def transfer(cls, type_id, _from, _to): CITypeAttributesCache.clean(type_id) from api.tasks.cmdb import ci_type_attribute_order_rebuild - ci_type_attribute_order_rebuild.apply_async(args=(type_id,), queue=CMDB_QUEUE) + ci_type_attribute_order_rebuild.apply_async(args=(type_id, current_user.uid), queue=CMDB_QUEUE) class CITypeTemplateManager(object): @@ -793,6 +874,12 @@ def __import(cls, data): for added_id in set(id2obj_dicts.keys()) - set(existed_ids): if cls == CIType: CITypeManager.add(**id2obj_dicts[added_id]) + elif cls == CITypeRelation: + CITypeRelationManager.add(id2obj_dicts[added_id].get('parent_id'), + id2obj_dicts[added_id].get('child_id'), + id2obj_dicts[added_id].get('relation_type_id'), + id2obj_dicts[added_id].get('constraint'), + ) else: cls.create(flush=True, **id2obj_dicts[added_id]) @@ -809,7 +896,7 @@ def __import(cls, data): ResourceTypeEnum.CI, permissions=[PermEnum.READ]) ACLManager().grant_resource_to_role(type_name, - g.user.username, + current_user.username, ResourceTypeEnum.CI) else: @@ -947,11 +1034,11 @@ def _import_auto_discovery_rules(rules): rule.pop("created_at", None) rule.pop("updated_at", None) - rule['uid'] = g.user.uid + rule['uid'] = current_user.uid try: AutoDiscoveryCITypeCRUD.add(**rule) - except: - pass + except Exception as e: + current_app.logger.warning("import auto discovery rules failed: {}".format(e)) def import_template(self, tpt): import time @@ -1016,7 +1103,7 @@ def export_template(): for ci_type in tpt['ci_types']: tpt['type2attributes'][ci_type['id']] = CITypeAttributeManager.get_attributes_by_type_id( - ci_type['id'], choice_web_hook_parse=False) + ci_type['id'], choice_web_hook_parse=False, choice_other_parse=False) tpt['type2attribute_group'][ci_type['id']] = CITypeAttributeGroupManager.get_by_type_id(ci_type['id']) @@ -1090,16 +1177,18 @@ def delete(_id): class CITypeTriggerManager(object): @staticmethod - def get(type_id): - return CITypeTrigger.get_by(type_id=type_id, to_dict=True) + def get(type_id, to_dict=True): + return CITypeTrigger.get_by(type_id=type_id, to_dict=to_dict) @staticmethod - def add(type_id, attr_id, notify): - CITypeTrigger.get_by(type_id=type_id, attr_id=attr_id) and abort(400, ErrFormat.ci_type_trigger_duplicate) + def add(type_id, attr_id, option): + for i in CITypeTrigger.get_by(type_id=type_id, attr_id=attr_id, to_dict=False): + if i.option == option: + return abort(400, ErrFormat.ci_type_trigger_duplicate) - not isinstance(notify, dict) and abort(400, ErrFormat.argument_invalid.format("notify")) + not isinstance(option, dict) and abort(400, ErrFormat.argument_invalid.format("option")) - trigger = CITypeTrigger.create(type_id=type_id, attr_id=attr_id, notify=notify) + trigger = CITypeTrigger.create(type_id=type_id, attr_id=attr_id, option=option) CITypeHistoryManager.add(CITypeOperateType.ADD_TRIGGER, type_id, @@ -1109,12 +1198,12 @@ def add(type_id, attr_id, notify): return trigger.to_dict() @staticmethod - def update(_id, notify): - existed = CITypeTrigger.get_by_id(_id) or \ - abort(404, ErrFormat.ci_type_trigger_not_found.format("id={}".format(_id))) + def update(_id, attr_id, option): + existed = (CITypeTrigger.get_by_id(_id) or + abort(404, ErrFormat.ci_type_trigger_not_found.format("id={}".format(_id)))) existed2 = existed.to_dict() - new = existed.update(notify=notify) + new = existed.update(attr_id=attr_id or None, option=option, filter_none=False) CITypeHistoryManager.add(CITypeOperateType.UPDATE_TRIGGER, existed.type_id, @@ -1125,8 +1214,8 @@ def update(_id, notify): @staticmethod def delete(_id): - existed = CITypeTrigger.get_by_id(_id) or \ - abort(404, ErrFormat.ci_type_trigger_not_found.format("id={}".format(_id))) + existed = (CITypeTrigger.get_by_id(_id) or + abort(404, ErrFormat.ci_type_trigger_not_found.format("id={}".format(_id)))) existed.soft_delete() @@ -1134,35 +1223,3 @@ def delete(_id): existed.type_id, trigger_id=_id, change=existed.to_dict()) - - @staticmethod - def waiting_cis(trigger): - now = datetime.datetime.today() - - delta_time = datetime.timedelta(days=(trigger.notify.get('before_days', 0) or 0)) - - attr = AttributeCache.get(trigger.attr_id) - - value_table = TableMap(attr=attr).table - - values = value_table.get_by(attr_id=attr.id, to_dict=False) - - result = [] - for v in values: - if isinstance(v.value, (datetime.date, datetime.datetime)) and \ - (v.value - delta_time).strftime('%Y%m%d') == now.strftime("%Y%m%d"): - result.append(v) - - return result - - @staticmethod - def trigger_notify(trigger, ci): - if trigger.notify.get('notify_at') == datetime.datetime.now().strftime("%H:%M") or \ - not trigger.notify.get('notify_at'): - from api.tasks.cmdb import trigger_notify - - trigger_notify.apply_async(args=(trigger.notify, ci.ci_id), queue=CMDB_QUEUE) - - return True - - return False diff --git a/cmdb-api/api/lib/cmdb/const.py b/cmdb-api/api/lib/cmdb/const.py index 118e053f..dc9497a0 100644 --- a/cmdb-api/api/lib/cmdb/const.py +++ b/cmdb-api/api/lib/cmdb/const.py @@ -12,6 +12,8 @@ class ValueTypeEnum(BaseEnum): DATE = "4" TIME = "5" JSON = "6" + PASSWORD = TEXT + LINK = TEXT class ConstraintEnum(BaseEnum): @@ -99,5 +101,7 @@ class AttributeDefaultValueEnum(BaseEnum): REDIS_PREFIX_CI = "ONE_CMDB" REDIS_PREFIX_CI_RELATION = "CMDB_CI_RELATION" +BUILTIN_KEYWORDS = {'id', '_id', 'ci_id', 'type', '_type', 'ci_type'} + L_TYPE = None L_CI = None diff --git a/cmdb-api/api/lib/cmdb/custom_dashboard.py b/cmdb-api/api/lib/cmdb/custom_dashboard.py index 8153eec3..133769ac 100644 --- a/cmdb-api/api/lib/cmdb/custom_dashboard.py +++ b/cmdb-api/api/lib/cmdb/custom_dashboard.py @@ -14,6 +14,14 @@ class CustomDashboardManager(object): def get(): return sorted(CustomDashboard.get_by(to_dict=True), key=lambda x: (x["category"], x['order'])) + @staticmethod + def preview(**kwargs): + from api.lib.cmdb.cache import CMDBCounterCache + + res = CMDBCounterCache.update(kwargs, flush=False) + + return res + @staticmethod def add(**kwargs): from api.lib.cmdb.cache import CMDBCounterCache @@ -23,9 +31,9 @@ def add(**kwargs): new = CustomDashboard.create(**kwargs) - CMDBCounterCache.update(new.to_dict()) + res = CMDBCounterCache.update(new.to_dict()) - return new + return new, res @staticmethod def update(_id, **kwargs): @@ -35,9 +43,9 @@ def update(_id, **kwargs): new = existed.update(**kwargs) - CMDBCounterCache.update(new.to_dict()) + res = CMDBCounterCache.update(new.to_dict()) - return new + return new, res @staticmethod def batch_update(id2options): diff --git a/cmdb-api/api/lib/cmdb/history.py b/cmdb-api/api/lib/cmdb/history.py index 43474ec2..6d1f5428 100644 --- a/cmdb-api/api/lib/cmdb/history.py +++ b/cmdb-api/api/lib/cmdb/history.py @@ -4,7 +4,7 @@ import json from flask import abort -from flask import g +from flask_login import current_user from api.extensions import db from api.lib.cmdb.cache import AttributeCache @@ -16,6 +16,7 @@ from api.models.cmdb import Attribute from api.models.cmdb import AttributeHistory from api.models.cmdb import CIRelationHistory +from api.models.cmdb import CITriggerHistory from api.models.cmdb import CITypeHistory from api.models.cmdb import CITypeTrigger from api.models.cmdb import CITypeUniqueConstraint @@ -176,8 +177,8 @@ def get_by_ci_id(ci_id): def get_record_detail(record_id): from api.lib.cmdb.ci import CIManager - record = OperationRecord.get_by_id(record_id) or \ - abort(404, ErrFormat.record_not_found.format("id={}".format(record_id))) + record = (OperationRecord.get_by_id(record_id) or + abort(404, ErrFormat.record_not_found.format("id={}".format(record_id)))) username = UserCache.get(record.uid).nickname or UserCache.get(record.uid).username timestamp = record.created_at.strftime("%Y-%m-%d %H:%M:%S") @@ -201,7 +202,7 @@ def get_record_detail(record_id): @staticmethod def add(record_id, ci_id, history_list, type_id=None, flush=False, commit=True): if record_id is None: - record = OperationRecord.create(uid=g.user.uid, type_id=type_id) + record = OperationRecord.create(uid=current_user.uid, type_id=type_id) record_id = record.id for attr_id, operate_type, old, new in history_list or []: @@ -220,7 +221,7 @@ def add(record_id, ci_id, history_list, type_id=None, flush=False, commit=True): class CIRelationHistoryManager(object): @staticmethod def add(rel_obj, operate_type=OperateType.ADD): - record = OperationRecord.create(uid=g.user.uid) + record = OperationRecord.create(uid=current_user.uid) CIRelationHistory.create(relation_id=rel_obj.id, record_id=record.id, @@ -279,10 +280,75 @@ def add(operate_type, type_id, attr_id=None, trigger_id=None, unique_constraint_ for _type_id in type_ids: payload = dict(operate_type=operate_type, type_id=_type_id, - uid=g.user.uid, + uid=current_user.uid, attr_id=attr_id, trigger_id=trigger_id, unique_constraint_id=unique_constraint_id, change=change) CITypeHistory.create(**payload) + + +class CITriggerHistoryManager(object): + @staticmethod + def get(page, page_size, type_id=None, trigger_id=None, operate_type=None): + query = CITriggerHistory.get_by(only_query=True) + if type_id: + query = query.filter(CITriggerHistory.type_id == type_id) + + if trigger_id: + query = query.filter(CITriggerHistory.trigger_id == trigger_id) + + if operate_type: + query = query.filter(CITriggerHistory.operate_type == operate_type) + + numfound = query.count() + + query = query.order_by(CITriggerHistory.id.desc()) + result = query.offset((page - 1) * page_size).limit(page_size) + result = [i.to_dict() for i in result] + for res in result: + if res.get('trigger_id'): + trigger = CITypeTrigger.get_by_id(res['trigger_id']) + res['trigger'] = trigger and trigger.to_dict() + + return numfound, result + + @staticmethod + def get_by_ci_id(ci_id): + res = db.session.query(CITriggerHistory, CITypeTrigger).join( + CITypeTrigger, CITypeTrigger.id == CITriggerHistory.trigger_id).filter( + CITriggerHistory.ci_id == ci_id).order_by(CITriggerHistory.id.desc()) + + result = [] + id2trigger = dict() + for i in res: + hist = i.CITriggerHistory + item = dict(is_ok=hist.is_ok, + operate_type=hist.operate_type, + notify=hist.notify, + trigger_id=hist.trigger_id, + trigger_name=hist.trigger_name, + webhook=hist.webhook, + created_at=hist.created_at.strftime('%Y-%m-%d %H:%M:%S'), + record_id=hist.record_id, + hid=hist.id + ) + if i.CITypeTrigger.id not in id2trigger: + id2trigger[i.CITypeTrigger.id] = i.CITypeTrigger.to_dict() + + result.append(item) + + return dict(items=result, id2trigger=id2trigger) + + @staticmethod + def add(operate_type, record_id, ci_id, trigger_id, trigger_name, is_ok=False, notify=None, webhook=None): + + CITriggerHistory.create(operate_type=operate_type, + record_id=record_id, + ci_id=ci_id, + trigger_id=trigger_id, + trigger_name=trigger_name, + is_ok=is_ok, + notify=notify, + webhook=webhook) diff --git a/cmdb-api/api/lib/cmdb/perms.py b/cmdb-api/api/lib/cmdb/perms.py index b4b149e9..3b5ad6f4 100644 --- a/cmdb-api/api/lib/cmdb/perms.py +++ b/cmdb-api/api/lib/cmdb/perms.py @@ -4,8 +4,8 @@ from flask import abort from flask import current_app -from flask import g from flask import request +from flask_login import current_user from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.resp_format import ErrFormat @@ -74,7 +74,7 @@ def get_by_ids(self, _ids, type_id=None): @classmethod def get_attr_filter(cls, type_id): - if is_app_admin('cmdb') or g.user.username in ('worker', 'cmdb_agent'): + if is_app_admin('cmdb') or current_user.username in ('worker', 'cmdb_agent'): return [] res2 = ACLManager('cmdb').get_resources(ResourceTypeEnum.CI_FILTER) @@ -160,7 +160,7 @@ def wrapper_has_perm(*args, **kwargs): resource = callback(resource) if current_app.config.get("USE_ACL") and resource: - if g.user.username == "worker" or g.user.username == "cmdb_agent": + if current_user.username == "worker" or current_user.username == "cmdb_agent": request.values['__is_admin'] = True return func(*args, **kwargs) diff --git a/cmdb-api/api/lib/cmdb/preference.py b/cmdb-api/api/lib/cmdb/preference.py index 2c07880c..eec09adf 100644 --- a/cmdb-api/api/lib/cmdb/preference.py +++ b/cmdb-api/api/lib/cmdb/preference.py @@ -7,7 +7,7 @@ import toposort from flask import abort from flask import current_app -from flask import g +from flask_login import current_user from api.extensions import db from api.lib.cmdb.attribute import AttributeManager @@ -36,11 +36,13 @@ class PreferenceManager(object): @staticmethod def get_types(instance=False, tree=False): types = db.session.query(PreferenceShowAttributes.type_id).filter( - PreferenceShowAttributes.uid == g.user.uid).filter( - PreferenceShowAttributes.deleted.is_(False)).group_by(PreferenceShowAttributes.type_id).all() \ - if instance else [] - tree_types = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=False) if tree else [] - type_ids = list(set([i.type_id for i in types + tree_types])) + PreferenceShowAttributes.uid == current_user.uid).filter( + PreferenceShowAttributes.deleted.is_(False)).group_by( + PreferenceShowAttributes.type_id).all() if instance else [] + + tree_types = PreferenceTreeView.get_by(uid=current_user.uid, to_dict=False) if tree else [] + type_ids = set([i.type_id for i in types + tree_types]) + return [CITypeCache.get(type_id).to_dict() for type_id in type_ids] @staticmethod @@ -62,7 +64,7 @@ def get_types2(instance=False, tree=False): PreferenceShowAttributes.deleted.is_(False)).group_by( PreferenceShowAttributes.uid, PreferenceShowAttributes.type_id) for i in types: - if i.uid == g.user.uid: + if i.uid == current_user.uid: result['self']['instance'].append(i.type_id) if str(i.created_at) > str(result['self']['type_id2subs_time'].get(i.type_id, "")): result['self']['type_id2subs_time'][i.type_id] = i.created_at @@ -72,7 +74,7 @@ def get_types2(instance=False, tree=False): if tree: types = PreferenceTreeView.get_by(to_dict=False) for i in types: - if i.uid == g.user.uid: + if i.uid == current_user.uid: result['self']['tree'].append(i.type_id) if str(i.created_at) > str(result['self']['type_id2subs_time'].get(i.type_id, "")): result['self']['type_id2subs_time'][i.type_id] = i.created_at @@ -91,7 +93,7 @@ def get_show_attributes(type_id): attrs = db.session.query(PreferenceShowAttributes, CITypeAttribute.order).join( CITypeAttribute, CITypeAttribute.attr_id == PreferenceShowAttributes.attr_id).filter( - PreferenceShowAttributes.uid == g.user.uid).filter( + PreferenceShowAttributes.uid == current_user.uid).filter( PreferenceShowAttributes.type_id == type_id).filter( PreferenceShowAttributes.deleted.is_(False)).filter(CITypeAttribute.deleted.is_(False)).filter( CITypeAttribute.type_id == type_id).all() @@ -114,13 +116,13 @@ def get_show_attributes(type_id): for i in result: if i["is_choice"]: i.update(dict(choice_value=AttributeManager.get_choice_values( - i["id"], i["value_type"], i["choice_web_hook"]))) + i["id"], i["value_type"], i["choice_web_hook"], i.get("choice_other")))) return is_subscribed, result @classmethod def create_or_update_show_attributes(cls, type_id, attr_order): - existed_all = PreferenceShowAttributes.get_by(type_id=type_id, uid=g.user.uid, to_dict=False) + existed_all = PreferenceShowAttributes.get_by(type_id=type_id, uid=current_user.uid, to_dict=False) for x, order in attr_order: if isinstance(x, list): _attr, is_fixed = x @@ -128,13 +130,13 @@ def create_or_update_show_attributes(cls, type_id, attr_order): _attr, is_fixed = x, False attr = AttributeCache.get(_attr) or abort(404, ErrFormat.attribute_not_found.format("id={}".format(_attr))) existed = PreferenceShowAttributes.get_by(type_id=type_id, - uid=g.user.uid, + uid=current_user.uid, attr_id=attr.id, first=True, to_dict=False) if existed is None: PreferenceShowAttributes.create(type_id=type_id, - uid=g.user.uid, + uid=current_user.uid, attr_id=attr.id, order=order, is_fixed=is_fixed) @@ -148,7 +150,7 @@ def create_or_update_show_attributes(cls, type_id, attr_order): @staticmethod def get_tree_view(): - res = PreferenceTreeView.get_by(uid=g.user.uid, to_dict=True) + res = PreferenceTreeView.get_by(uid=current_user.uid, to_dict=True) for item in res: if item["levels"]: ci_type = CITypeCache.get(item['type_id']).to_dict() @@ -176,14 +178,14 @@ def create_or_update_tree_view(type_id, levels): if i == attr.id or i == attr.name or i == attr.alias: levels[idx] = attr.id - existed = PreferenceTreeView.get_by(uid=g.user.uid, type_id=type_id, to_dict=False, first=True) + existed = PreferenceTreeView.get_by(uid=current_user.uid, type_id=type_id, to_dict=False, first=True) if existed is not None: if not levels: existed.soft_delete() return existed return existed.update(levels=levels) elif levels: - return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=g.user.uid) + return PreferenceTreeView.create(levels=levels, type_id=type_id, uid=current_user.uid) @staticmethod def get_relation_view(): @@ -254,7 +256,7 @@ def create_or_update_relation_view(cls, name, cr_ids, is_public=False): existed = PreferenceRelationView.get_by(name=name, to_dict=False, first=True) current_app.logger.debug(existed) if existed is None: - PreferenceRelationView.create(name=name, cr_ids=cr_ids, uid=g.user.uid, is_public=is_public) + PreferenceRelationView.create(name=name, cr_ids=cr_ids, uid=current_user.uid, is_public=is_public) if current_app.config.get("USE_ACL"): ACLManager().add_resource(name, ResourceTypeEnum.RELATION_VIEW) @@ -278,7 +280,7 @@ def delete_relation_view(name): @staticmethod def get_search_option(**kwargs): query = PreferenceSearchOption.get_by(only_query=True) - query = query.filter(PreferenceSearchOption.uid == g.user.uid) + query = query.filter(PreferenceSearchOption.uid == current_user.uid) for k in kwargs: if hasattr(PreferenceSearchOption, k) and kwargs[k]: @@ -288,9 +290,9 @@ def get_search_option(**kwargs): @staticmethod def add_search_option(**kwargs): - kwargs['uid'] = g.user.uid + kwargs['uid'] = current_user.uid - existed = PreferenceSearchOption.get_by(uid=g.user.uid, + existed = PreferenceSearchOption.get_by(uid=current_user.uid, name=kwargs.get('name'), prv_id=kwargs.get('prv_id'), ptv_id=kwargs.get('ptv_id'), @@ -306,10 +308,10 @@ def update_search_option(_id, **kwargs): existed = PreferenceSearchOption.get_by_id(_id) or abort(404, ErrFormat.preference_search_option_not_found) - if g.user.uid != existed.uid: + if current_user.uid != existed.uid: return abort(400, ErrFormat.no_permission2) - other = PreferenceSearchOption.get_by(uid=g.user.uid, + other = PreferenceSearchOption.get_by(uid=current_user.uid, name=kwargs.get('name'), prv_id=kwargs.get('prv_id'), ptv_id=kwargs.get('ptv_id'), @@ -324,7 +326,7 @@ def update_search_option(_id, **kwargs): def delete_search_option(_id): existed = PreferenceSearchOption.get_by_id(_id) or abort(404, ErrFormat.preference_search_option_not_found) - if g.user.uid != existed.uid: + if current_user.uid != existed.uid: return abort(400, ErrFormat.no_permission2) existed.soft_delete() diff --git a/cmdb-api/api/lib/cmdb/query_sql.py b/cmdb-api/api/lib/cmdb/query_sql.py index c84fd649..f5c598bb 100644 --- a/cmdb-api/api/lib/cmdb/query_sql.py +++ b/cmdb-api/api/lib/cmdb/query_sql.py @@ -42,7 +42,7 @@ FACET_QUERY = """ SELECT {0}.value, - count({0}.ci_id) + count(distinct({0}.ci_id)) FROM {0} INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id WHERE {0}.attr_id={2:d} diff --git a/cmdb-api/api/lib/cmdb/relation_type.py b/cmdb-api/api/lib/cmdb/relation_type.py index 1fd6c9bf..2eb58617 100644 --- a/cmdb-api/api/lib/cmdb/relation_type.py +++ b/cmdb-api/api/lib/cmdb/relation_type.py @@ -24,21 +24,21 @@ def get_pairs(cls): @staticmethod def add(name): - RelationType.get_by(name=name, first=True, to_dict=False) and \ - abort(400, ErrFormat.relation_type_exists.format(name)) + RelationType.get_by(name=name, first=True, to_dict=False) and abort( + 400, ErrFormat.relation_type_exists.format(name)) return RelationType.create(name=name) @staticmethod def update(rel_id, name): - existed = RelationType.get_by_id(rel_id) or \ - abort(404, ErrFormat.relation_type_not_found.format("id={}".format(rel_id))) + existed = RelationType.get_by_id(rel_id) or abort( + 404, ErrFormat.relation_type_not_found.format("id={}".format(rel_id))) return existed.update(name=name) @staticmethod def delete(rel_id): - existed = RelationType.get_by_id(rel_id) or \ - abort(404, ErrFormat.relation_type_not_found.format("id={}".format(rel_id))) + existed = RelationType.get_by_id(rel_id) or abort( + 404, ErrFormat.relation_type_not_found.format("id={}".format(rel_id))) existed.soft_delete() diff --git a/cmdb-api/api/lib/cmdb/resp_format.py b/cmdb-api/api/lib/cmdb/resp_format.py index ed21e86d..ef040beb 100644 --- a/cmdb-api/api/lib/cmdb/resp_format.py +++ b/cmdb-api/api/lib/cmdb/resp_format.py @@ -10,6 +10,8 @@ class ErrFormat(CommonErrFormat): argument_file_not_found = "文件似乎并未上传" attribute_not_found = "属性 {} 不存在!" + attribute_is_unique_id = "该属性是模型的唯一标识,不能被删除!" + attribute_is_ref_by_type = "该属性被模型 {} 引用, 不能删除!" attribute_value_type_cannot_change = "属性的值类型不允许修改!" attribute_list_value_cannot_change = "多值不被允许修改!" attribute_index_cannot_change = "修改索引 非管理员不被允许!" @@ -19,8 +21,9 @@ class ErrFormat(CommonErrFormat): add_attribute_failed = "创建属性 {} 失败!" update_attribute_failed = "修改属性 {} 失败!" cannot_edit_attribute = "您没有权限修改该属性!" - cannot_delete_attribute = "您没有权限删除该属性!" + cannot_delete_attribute = "目前只允许 属性创建人、管理员 删除属性!" attribute_name_cannot_be_builtin = "属性字段名不能是内置字段: id, _id, ci_id, type, _type, ci_type" + attribute_choice_other_invalid = "预定义值: 其他模型请求参数不合法!" ci_not_found = "CI {} 不存在" unique_constraint = "多属性联合唯一校验不通过: {}" @@ -36,6 +39,7 @@ class ErrFormat(CommonErrFormat): unique_key_not_define = "主键未定义或者已被删除" only_owner_can_delete = "只有创建人才能删除它!" ci_exists_and_cannot_delete_type = "因为CI已经存在,不能删除模型" + ci_relation_view_exists_and_cannot_delete_type = "因为关系视图 {} 引用了该模型,不能删除模型" ci_type_group_not_found = "模型分组 {} 不存在" ci_type_group_exists = "模型分组 {} 已经存在" ci_type_relation_not_found = "模型关系 {} 不存在" @@ -91,3 +95,6 @@ class ErrFormat(CommonErrFormat): ci_filter_perm_cannot_or_query = "CI过滤授权 暂时不支持 或 查询" ci_filter_perm_attr_no_permission = "您没有属性 {} 的操作权限!" ci_filter_perm_ci_no_permission = "您没有该CI的操作权限!" + + password_save_failed = "保存密码失败: {}" + password_load_failed = "获取密码失败: {}" diff --git a/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py b/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py index 78e43e75..24aa0cb3 100644 --- a/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py +++ b/cmdb-api/api/lib/cmdb/search/ci/db/query_sql.py @@ -7,6 +7,7 @@ attr.alias AS attr_alias, attr.value_type, attr.is_list, + attr.is_password, c_cis.type_id, {0}.ci_id, {0}.attr_id, @@ -26,7 +27,8 @@ A.attr_alias, A.value, A.value_type, - A.is_list + A.is_list, + A.is_password FROM ({1}) AS A {0} ORDER BY A.ci_id; @@ -43,7 +45,7 @@ FACET_QUERY = """ SELECT {0}.value, - count({0}.ci_id) + count(distinct {0}.ci_id) FROM {0} INNER JOIN ({1}) AS F ON F.ci_id={0}.ci_id WHERE {0}.attr_id={2:d} diff --git a/cmdb-api/api/lib/cmdb/search/ci/db/search.py b/cmdb-api/api/lib/cmdb/search/ci/db/search.py index 48e2fb9b..206e9214 100644 --- a/cmdb-api/api/lib/cmdb/search/ci/db/search.py +++ b/cmdb-api/api/lib/cmdb/search/ci/db/search.py @@ -1,4 +1,4 @@ -# -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- from __future__ import unicode_literals @@ -7,8 +7,10 @@ import time from flask import current_app -from flask import g +from flask_login import current_user from jinja2 import Template +from sqlalchemy import text + from api.extensions import db from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.cache import CITypeCache @@ -27,6 +29,7 @@ from api.lib.cmdb.search.ci.db.query_sql import QUERY_CI_BY_TYPE from api.lib.cmdb.search.ci.db.query_sql import QUERY_UNION_CI_ATTRIBUTE_IS_NULL from api.lib.cmdb.utils import TableMap +from api.lib.cmdb.utils import ValueTypeMap from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import is_app_admin from api.lib.utils import handle_arg_list @@ -105,7 +108,7 @@ def _type_query_handler(self, v, queries): ci_filter = self.type2filter_perms[ci_type.id].get('ci_filter') if ci_filter: sub = [] - ci_filter = Template(ci_filter).render(user=g.user) + ci_filter = Template(ci_filter).render(user=current_user) for i in ci_filter.split(','): if i.startswith("~") and not sub: queries.append(i) @@ -140,6 +143,10 @@ def _id_query_handler(v): @staticmethod def _in_query_handler(attr, v, is_not): new_v = v[1:-1].split(";") + + if attr.value_type == ValueTypeEnum.DATE: + new_v = ["{} 00:00:00".format(i) for i in new_v if len(i) == 10] + table_name = TableMap(attr=attr).table_name in_query = " OR {0}.value ".format(table_name).join(['{0} "{1}"'.format( "NOT LIKE" if is_not else "LIKE", @@ -150,6 +157,11 @@ def _in_query_handler(attr, v, is_not): @staticmethod def _range_query_handler(attr, v, is_not): start, end = [x.strip() for x in v[1:-1].split("_TO_")] + + if attr.value_type == ValueTypeEnum.DATE: + start = "{} 00:00:00".format(start) if len(start) == 10 else start + end = "{} 00:00:00".format(end) if len(end) == 10 else end + table_name = TableMap(attr=attr).table_name range_query = "{0} '{1}' AND '{2}'".format( "NOT BETWEEN" if is_not else "BETWEEN", @@ -161,8 +173,14 @@ def _range_query_handler(attr, v, is_not): def _comparison_query_handler(attr, v): table_name = TableMap(attr=attr).table_name if v.startswith(">=") or v.startswith("<="): + if attr.value_type == ValueTypeEnum.DATE and len(v[2:]) == 10: + v = "{} 00:00:00".format(v) + comparison_query = "{0} '{1}'".format(v[:2], v[2:].replace("*", "%")) else: + if attr.value_type == ValueTypeEnum.DATE and len(v[1:]) == 10: + v = "{} 00:00:00".format(v) + comparison_query = "{0} '{1}'".format(v[0], v[1:].replace("*", "%")) _query_sql = QUERY_CI_BY_ATTR_NAME.format(table_name, attr.id, comparison_query) return _query_sql @@ -238,16 +256,14 @@ def __sort_by_field(self, field, sort_type, query_sql): attr_id = attr.id table_name = TableMap(attr=attr).table_name - _v_query_sql = """SELECT {0}.ci_id, {1}.value + _v_query_sql = """SELECT {0}.ci_id, {1}.value FROM ({2}) AS {0} INNER JOIN {1} ON {1}.ci_id = {0}.ci_id WHERE {1}.attr_id = {3}""".format("ALIAS", table_name, query_sql, attr_id) new_table = _v_query_sql if self.only_type_query or not self.type_id_list: - return "SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id " \ - "FROM ({0}) AS C " \ - "ORDER BY C.value {2} " \ - "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count) + return ("SELECT SQL_CALC_FOUND_ROWS DISTINCT C.ci_id FROM ({0}) AS C ORDER BY C.value {2} " + "LIMIT {1:d}, {3};".format(new_table, (self.page - 1) * self.count, sort_type, self.count)) elif self.type_id_list: self.query_sql = """SELECT C.ci_id @@ -286,7 +302,7 @@ def _wrap_sql(operator, alias, _query_sql, query_sql): query_sql = "SELECT * FROM ({0}) as {1} UNION ALL ({2})".format(query_sql, alias, _query_sql) elif operator == "~": - query_sql = """SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} USING(ci_id) + query_sql = """SELECT * FROM ({0}) as {1} LEFT JOIN ({2}) as {3} USING(ci_id) WHERE {3}.ci_id is NULL""".format(query_sql, alias, _query_sql, alias + "A") return query_sql @@ -296,8 +312,8 @@ def _execute_sql(self, query_sql): start = time.time() execute = db.session.execute - current_app.logger.debug(v_query_sql) - res = execute(v_query_sql).fetchall() + # current_app.logger.debug(v_query_sql) + res = execute(text(v_query_sql)).fetchall() end_time = time.time() current_app.logger.debug("query ci ids time is: {0}".format(end_time - start)) @@ -355,7 +371,7 @@ def __confirm_type_first(self, queries): else: result.append(q) - _is_app_admin = is_app_admin('cmdb') or g.user.username == "worker" + _is_app_admin = is_app_admin('cmdb') or current_user.username == "worker" if result and not has_type and not _is_app_admin: type_q = self.__get_types_has_read() if id_query: @@ -392,6 +408,9 @@ def __query_by_attr(self, q, queries, alias): is_not = True if operator == "|~" else False + if field_type == ValueTypeEnum.DATE and len(v) == 10: + v = "{} 00:00:00".format(v) + # in query if v.startswith("(") and v.endswith(")"): _query_sql = self._in_query_handler(attr, v, is_not) @@ -507,15 +526,15 @@ def _facet_build(self): if k: table_name = TableMap(attr=attr).table_name query_sql = FACET_QUERY.format(table_name, self.query_sql, attr.id) - # current_app.logger.debug(query_sql) - result = db.session.execute(query_sql).fetchall() + result = db.session.execute(text(query_sql)).fetchall() facet[k] = result facet_result = dict() for k, v in facet.items(): if not k.startswith('_'): - a = getattr(AttributeCache.get(k), self.ret_key) - facet_result[a] = [(f[0], f[1], a) for f in v] + attr = AttributeCache.get(k) + a = getattr(attr, self.ret_key) + facet_result[a] = [(ValueTypeMap.serialize[attr.value_type](f[0]), f[1], a) for f in v] return facet_result diff --git a/cmdb-api/api/lib/cmdb/search/ci/es/search.py b/cmdb-api/api/lib/cmdb/search/ci/es/search.py index e83cf690..235e5bb7 100644 --- a/cmdb-api/api/lib/cmdb/search/ci/es/search.py +++ b/cmdb-api/api/lib/cmdb/search/ci/es/search.py @@ -297,8 +297,8 @@ def _sort_build(self): if not attr: raise SearchError(ErrFormat.attribute_not_found.format(field)) - sort_by = "{0}.keyword".format(field) \ - if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field + sort_by = ("{0}.keyword".format(field) + if attr.value_type not in (ValueTypeEnum.INT, ValueTypeEnum.FLOAT) else field) sorts.append({sort_by: {"order": sort_type}}) self.query.update(dict(sort=sorts)) diff --git a/cmdb-api/api/lib/cmdb/search/ci_relation/search.py b/cmdb-api/api/lib/cmdb/search/ci_relation/search.py index 3a22f3be..0d47e3d4 100644 --- a/cmdb-api/api/lib/cmdb/search/ci_relation/search.py +++ b/cmdb-api/api/lib/cmdb/search/ci_relation/search.py @@ -35,7 +35,7 @@ def __init__(self, root_id, self.sort = sort or ("ci_id" if current_app.config.get("USE_ES") else None) self.root_id = root_id - self.level = level + self.level = level or 0 self.reverse = reverse def _get_ids(self): @@ -104,16 +104,22 @@ def search(self): ci_ids=merge_ids).search() def statistics(self, type_ids): + self.level = int(self.level) _tmp = [] ids = [self.root_id] if not isinstance(self.root_id, list) else self.root_id - for l in range(0, int(self.level)): - if not l: - _tmp = list(map(lambda x: list(json.loads(x).items()), - [i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []])) + for lv in range(0, self.level): + if not lv: + if type_ids and lv == self.level - 1: + _tmp = list(map(lambda x: [i for i in x if i[1] in type_ids], + (map(lambda x: list(json.loads(x).items()), + [i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []])))) + else: + _tmp = list(map(lambda x: list(json.loads(x).items()), + [i or '{}' for i in rd.get(ids, REDIS_PREFIX_CI_RELATION) or []])) else: for idx, item in enumerate(_tmp): if item: - if type_ids and l == self.level - 1: + if type_ids and lv == self.level - 1: __tmp = list( map(lambda x: [(_id, type_id) for _id, type_id in json.loads(x).items() if type_id in type_ids], diff --git a/cmdb-api/api/lib/cmdb/utils.py b/cmdb-api/api/lib/cmdb/utils.py index 1ea82af6..9ee096e5 100644 --- a/cmdb-api/api/lib/cmdb/utils.py +++ b/cmdb-api/api/lib/cmdb/utils.py @@ -4,14 +4,16 @@ import datetime import json +import re import six -from markupsafe import escape import api.models.cmdb as model from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.const import ValueTypeEnum +TIME_RE = re.compile(r"^20|21|22|23|[0-1]\d:[0-5]\d:[0-5]\d$") + def string2int(x): return int(float(x)) @@ -19,7 +21,7 @@ def string2int(x): def str2datetime(x): try: - return datetime.datetime.strptime(x, "%Y-%m-%d") + return datetime.datetime.strptime(x, "%Y-%m-%d").date() except ValueError: pass @@ -30,8 +32,8 @@ class ValueTypeMap(object): deserialize = { ValueTypeEnum.INT: string2int, ValueTypeEnum.FLOAT: float, - ValueTypeEnum.TEXT: lambda x: escape(x).encode('utf-8').decode('utf-8'), - ValueTypeEnum.TIME: lambda x: escape(x).encode('utf-8').decode('utf-8'), + ValueTypeEnum.TEXT: lambda x: x, + ValueTypeEnum.TIME: lambda x: TIME_RE.findall(x)[0], ValueTypeEnum.DATETIME: str2datetime, ValueTypeEnum.DATE: str2datetime, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, @@ -42,8 +44,8 @@ class ValueTypeMap(object): ValueTypeEnum.FLOAT: float, ValueTypeEnum.TEXT: lambda x: x if isinstance(x, six.string_types) else str(x), ValueTypeEnum.TIME: lambda x: x if isinstance(x, six.string_types) else str(x), - ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d"), - ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S"), + ValueTypeEnum.DATE: lambda x: x.strftime("%Y-%m-%d") if not isinstance(x, six.string_types) else x, + ValueTypeEnum.DATETIME: lambda x: x.strftime("%Y-%m-%d %H:%M:%S") if not isinstance(x, six.string_types) else x, ValueTypeEnum.JSON: lambda x: json.loads(x) if isinstance(x, six.string_types) and x else x, } @@ -61,15 +63,13 @@ class ValueTypeMap(object): ValueTypeEnum.INT: model.IntegerChoice, ValueTypeEnum.FLOAT: model.FloatChoice, ValueTypeEnum.TEXT: model.TextChoice, + ValueTypeEnum.TIME: model.TextChoice, + ValueTypeEnum.DATE: model.TextChoice, + ValueTypeEnum.DATETIME: model.TextChoice, } table = { - ValueTypeEnum.INT: model.CIValueInteger, ValueTypeEnum.TEXT: model.CIValueText, - ValueTypeEnum.DATETIME: model.CIValueDateTime, - ValueTypeEnum.DATE: model.CIValueDateTime, - ValueTypeEnum.TIME: model.CIValueText, - ValueTypeEnum.FLOAT: model.CIValueFloat, ValueTypeEnum.JSON: model.CIValueJson, 'index_{0}'.format(ValueTypeEnum.INT): model.CIIndexValueInteger, 'index_{0}'.format(ValueTypeEnum.TEXT): model.CIIndexValueText, @@ -81,12 +81,7 @@ class ValueTypeMap(object): } table_name = { - ValueTypeEnum.INT: 'c_value_integers', ValueTypeEnum.TEXT: 'c_value_texts', - ValueTypeEnum.DATETIME: 'c_value_datetime', - ValueTypeEnum.DATE: 'c_value_datetime', - ValueTypeEnum.TIME: 'c_value_texts', - ValueTypeEnum.FLOAT: 'c_value_floats', ValueTypeEnum.JSON: 'c_value_json', 'index_{0}'.format(ValueTypeEnum.INT): 'c_value_index_integers', 'index_{0}'.format(ValueTypeEnum.TEXT): 'c_value_index_texts', @@ -104,7 +99,7 @@ class ValueTypeMap(object): ValueTypeEnum.DATE: 'text', ValueTypeEnum.TIME: 'text', ValueTypeEnum.FLOAT: 'float', - ValueTypeEnum.JSON: 'object' + ValueTypeEnum.JSON: 'object', } @@ -117,8 +112,13 @@ def __init__(self, attr_name=None, attr=None, is_index=None): @property def table(self): attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr - if self.is_index is None: + if attr.is_password or attr.is_link: + self.is_index = False + elif attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON}: + self.is_index = True + elif self.is_index is None: self.is_index = attr.is_index + i = "index_{0}".format(attr.value_type) if self.is_index else attr.value_type return ValueTypeMap.table.get(i) @@ -126,8 +126,13 @@ def table(self): @property def table_name(self): attr = AttributeCache.get(self.attr_name) if not self.attr else self.attr - if self.is_index is None: + if attr.is_password or attr.is_link: + self.is_index = False + elif attr.value_type not in {ValueTypeEnum.TEXT, ValueTypeEnum.JSON}: + self.is_index = True + elif self.is_index is None: self.is_index = attr.is_index + i = "index_{0}".format(attr.value_type) if self.is_index else attr.value_type return ValueTypeMap.table_name.get(i) diff --git a/cmdb-api/api/lib/cmdb/value.py b/cmdb-api/api/lib/cmdb/value.py index 5b1c25d3..7a2e6f4c 100644 --- a/cmdb-api/api/lib/cmdb/value.py +++ b/cmdb-api/api/lib/cmdb/value.py @@ -18,7 +18,6 @@ from api.lib.cmdb.attribute import AttributeManager from api.lib.cmdb.cache import AttributeCache from api.lib.cmdb.cache import CITypeAttributeCache -from api.lib.cmdb.const import ExistPolicy from api.lib.cmdb.const import OperateType from api.lib.cmdb.const import ValueTypeEnum from api.lib.cmdb.history import AttributeHistoryManger @@ -67,9 +66,10 @@ def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_ma use_master=use_master, to_dict=False) field_name = getattr(attr, ret_key) - if attr.is_list: res[field_name] = [ValueTypeMap.serialize[attr.value_type](i.value) for i in rs] + elif attr.is_password and rs: + res[field_name] = '******' if rs[0].value else '' else: res[field_name] = ValueTypeMap.serialize[attr.value_type](rs[0].value) if rs else None @@ -80,9 +80,10 @@ def get_attr_values(self, fields, ci_id, ret_key="name", unique_key=None, use_ma return res @staticmethod - def __deserialize_value(value_type, value): + def _deserialize_value(value_type, value): if not value: return value + deserialize = ValueTypeMap.deserialize[value_type] try: v = deserialize(value) @@ -91,13 +92,13 @@ def __deserialize_value(value_type, value): return abort(400, ErrFormat.attribute_value_invalid.format(value)) @staticmethod - def __check_is_choice(attr, value_type, value): - choice_values = AttributeManager.get_choice_values(attr.id, value_type, attr.choice_web_hook) + def _check_is_choice(attr, value_type, value): + choice_values = AttributeManager.get_choice_values(attr.id, value_type, attr.choice_web_hook, attr.choice_other) if str(value) not in list(map(str, [i[0] for i in choice_values])): return abort(400, ErrFormat.not_in_choice_values.format(value)) @staticmethod - def __check_is_unique(value_table, attr, ci_id, type_id, value): + def _check_is_unique(value_table, attr, ci_id, type_id, value): existed = db.session.query(value_table.attr_id).join(CI, CI.id == value_table.ci_id).filter( CI.type_id == type_id).filter( value_table.attr_id == attr.id).filter(value_table.deleted.is_(False)).filter( @@ -106,20 +107,20 @@ def __check_is_unique(value_table, attr, ci_id, type_id, value): existed and abort(400, ErrFormat.attribute_value_unique_required.format(attr.alias, value)) @staticmethod - def __check_is_required(type_id, attr, value, type_attr=None): + def _check_is_required(type_id, attr, value, type_attr=None): type_attr = type_attr or CITypeAttributeCache.get(type_id, attr.id) if type_attr and type_attr.is_required and not value and value != 0: return abort(400, ErrFormat.attribute_value_required.format(attr.alias)) def _validate(self, attr, value, value_table, ci=None, type_id=None, ci_id=None, type_attr=None): ci = ci or {} - v = self.__deserialize_value(attr.value_type, value) + v = self._deserialize_value(attr.value_type, value) - attr.is_choice and value and self.__check_is_choice(attr, attr.value_type, v) - attr.is_unique and self.__check_is_unique( + attr.is_choice and value and self._check_is_choice(attr, attr.value_type, v) + attr.is_unique and self._check_is_unique( value_table, attr, ci and ci.id or ci_id, ci and ci.type_id or type_id, v) - self.__check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr) + self._check_is_required(ci and ci.type_id or type_id, attr, v, type_attr=type_attr) if v == "" and attr.value_type not in (ValueTypeEnum.TEXT,): v = None @@ -131,20 +132,20 @@ def _write_change(ci_id, attr_id, operate_type, old, new, record_id, type_id): return AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id) @staticmethod - def _write_change2(changed): - record_id = None + def write_change2(changed, record_id=None): for ci_id, attr_id, operate_type, old, new, type_id in changed: record_id = AttributeHistoryManger.add(record_id, ci_id, [(attr_id, operate_type, old, new)], type_id, commit=False, flush=False) try: db.session.commit() except Exception as e: + db.session.rollback() current_app.logger.error("write change failed: {}".format(str(e))) return record_id @staticmethod - def __compute_attr_value_from_expr(expr, ci_dict): + def _compute_attr_value_from_expr(expr, ci_dict): t = jinja2.Template(expr).render(ci_dict) try: @@ -154,7 +155,7 @@ def __compute_attr_value_from_expr(expr, ci_dict): return t @staticmethod - def __compute_attr_value_from_script(script, ci_dict): + def _compute_attr_value_from_script(script, ci_dict): script = jinja2.Template(script).render(ci_dict) script_f = tempfile.NamedTemporaryFile(delete=False, suffix=".py") @@ -183,22 +184,22 @@ def _jinja2_parse(content): return [var for var in schema.get("properties")] - def _compute_attr_value(self, attr, payload, ci): - attrs = self._jinja2_parse(attr['compute_expr']) if attr.get('compute_expr') else \ - self._jinja2_parse(attr['compute_script']) + def _compute_attr_value(self, attr, payload, ci_id): + attrs = (self._jinja2_parse(attr['compute_expr']) if attr.get('compute_expr') + else self._jinja2_parse(attr['compute_script'])) not_existed = [i for i in attrs if i not in payload] - if ci is not None: - payload.update(self.get_attr_values(not_existed, ci.id)) + if ci_id is not None: + payload.update(self.get_attr_values(not_existed, ci_id)) if attr['compute_expr']: - return self.__compute_attr_value_from_expr(attr['compute_expr'], payload) + return self._compute_attr_value_from_expr(attr['compute_expr'], payload) elif attr['compute_script']: - return self.__compute_attr_value_from_script(attr['compute_script'], payload) + return self._compute_attr_value_from_script(attr['compute_script'], payload) def handle_ci_compute_attributes(self, ci_dict, computed_attrs, ci): payload = copy.deepcopy(ci_dict) for attr in computed_attrs: - computed_value = self._compute_attr_value(attr, payload, ci) + computed_value = self._compute_attr_value(attr, payload, ci and ci.id) if computed_value is not None: ci_dict[attr['name']] = computed_value @@ -220,7 +221,7 @@ def valid_attr_value(self, ci_dict, type_id, ci_id, name2attr, alias2attr=None, for i in handle_arg_list(value)] ci_dict[key] = value_list if not value_list: - self.__check_is_required(type_id, attr, '') + self._check_is_required(type_id, attr, '') else: value = self._validate(attr, value, value_table, ci=None, type_id=type_id, ci_id=ci_id, @@ -234,7 +235,7 @@ def valid_attr_value(self, ci_dict, type_id, ci_id, name2attr, alias2attr=None, return key2attr - def create_or_update_attr_value2(self, ci, ci_dict, key2attr): + def create_or_update_attr_value(self, ci, ci_dict, key2attr): """ add or update attribute value, then write history :param ci: instance object @@ -283,69 +284,9 @@ def create_or_update_attr_value2(self, ci, ci_dict, key2attr): except Exception as e: db.session.rollback() current_app.logger.warning(str(e)) - return abort(400, ErrFormat.attribute_value_unknown_error.format(str(e))) - - return self._write_change2(changed) - - def create_or_update_attr_value(self, key, value, ci, _no_attribute_policy=ExistPolicy.IGNORE, record_id=None): - """ - add or update attribute value, then write history - :param key: id, name or alias - :param value: - :param ci: instance object - :param _no_attribute_policy: ignore or reject - :param record_id: op record - :return: - """ - attr = self._get_attr(key) - if attr is None: - if _no_attribute_policy == ExistPolicy.IGNORE: - return - if _no_attribute_policy == ExistPolicy.REJECT: - return abort(400, ErrFormat.attribute_not_found.format(key)) - - value_table = TableMap(attr=attr).table - - try: - if attr.is_list: - value_list = [self._validate(attr, i, value_table, ci) for i in handle_arg_list(value)] - if not value_list: - self.__check_is_required(ci.type_id, attr, '') - - existed_attrs = value_table.get_by(attr_id=attr.id, ci_id=ci.id, to_dict=False) - existed_values = [i.value for i in existed_attrs] - added = set(value_list) - set(existed_values) - deleted = set(existed_values) - set(value_list) - for v in added: - value_table.create(ci_id=ci.id, attr_id=attr.id, value=v) - record_id = self._write_change(ci.id, attr.id, OperateType.ADD, None, v, record_id, ci.type_id) - - for v in deleted: - existed_attr = existed_attrs[existed_values.index(v)] - existed_attr.delete() - record_id = self._write_change(ci.id, attr.id, OperateType.DELETE, v, None, record_id, ci.type_id) - else: - value = self._validate(attr, value, value_table, ci) - existed_attr = value_table.get_by(attr_id=attr.id, ci_id=ci.id, first=True, to_dict=False) - existed_value = existed_attr and existed_attr.value - if existed_value is None and value is not None: - value_table.create(ci_id=ci.id, attr_id=attr.id, value=value) + return abort(400, ErrFormat.attribute_value_unknown_error.format(e.args[0])) - record_id = self._write_change(ci.id, attr.id, OperateType.ADD, None, value, record_id, ci.type_id) - else: - if existed_value != value: - if value is None: - existed_attr.delete() - else: - existed_attr.update(value=value) - - record_id = self._write_change(ci.id, attr.id, OperateType.UPDATE, - existed_value, value, record_id, ci.type_id) - - return record_id - except Exception as e: - current_app.logger.warning(str(e)) - return abort(400, ErrFormat.attribute_value_invalid2.format("{}({})".format(attr.alias, attr.name), value)) + return self.write_change2(changed) @staticmethod def delete_attr_value(attr_id, ci_id): diff --git a/cmdb-api/api/lib/common_setting/acl.py b/cmdb-api/api/lib/common_setting/acl.py index 163a3732..67774cec 100644 --- a/cmdb-api/api/lib/common_setting/acl.py +++ b/cmdb-api/api/lib/common_setting/acl.py @@ -1,9 +1,11 @@ # -*- coding:utf-8 -*- -from flask import abort from flask import current_app from api.lib.common_setting.resp_format import ErrFormat +from api.lib.perm.acl.app import AppCRUD from api.lib.perm.acl.cache import RoleCache, AppCache +from api.lib.perm.acl.permission import PermissionCRUD +from api.lib.perm.acl.resource import ResourceTypeCRUD, ResourceCRUD from api.lib.perm.acl.role import RoleCRUD, RoleRelationCRUD from api.lib.perm.acl.user import UserCRUD @@ -78,19 +80,64 @@ def edit_role(_id, payload): return role.to_dict() @staticmethod - def delete_role(_id, payload): + def delete_role(_id): RoleCRUD.delete_role(_id) return dict(rid=_id) def get_user_info(self, username): from api.lib.perm.acl.acl import ACLManager as ACL user_info = ACL().get_user_info(username, self.app_name) - result = dict(name=user_info.get('nickname') or username, - username=user_info.get('username') or username, - email=user_info.get('email'), - uid=user_info.get('uid'), - rid=user_info.get('rid'), - role=dict(permissions=user_info.get('parents')), - avatar=user_info.get('avatar')) + result = dict( + name=user_info.get('nickname') or username, + username=user_info.get('username') or username, + email=user_info.get('email'), + uid=user_info.get('uid'), + rid=user_info.get('rid'), + role=dict(permissions=user_info.get('parents')), + avatar=user_info.get('avatar') + ) return result + + def validate_app(self): + return AppCache.get(self.app_name) + + def get_all_resources_types(self, q=None, page=1, page_size=999999): + app_id = self.validate_app().id + numfound, res, id2perms = ResourceTypeCRUD.search(q, app_id, page, page_size) + + return dict( + numfound=numfound, + groups=[i.to_dict() for i in res], + id2perms=id2perms + ) + + def create_resources_type(self, payload): + payload['app_id'] = self.validate_app().id + rt = ResourceTypeCRUD.add(**payload) + + return rt.to_dict() + + def update_resources_type(self, _id, payload): + rt = ResourceTypeCRUD.update(_id, **payload) + + return rt.to_dict() + + def create_resource(self, payload): + payload['app_id'] = self.validate_app().id + resource = ResourceCRUD.add(**payload) + + return resource.to_dict() + + def get_resource_by_type(self, q, u, rt_id, page=1, page_size=999999): + numfound, res = ResourceCRUD.search(q, u, self.validate_app().id, rt_id, page, page_size) + return res + + def grant_resource(self, rid, resource_id, perms): + PermissionCRUD.grant(rid, perms, resource_id=resource_id, group_id=None) + + @staticmethod + def create_app(payload): + rt = AppCRUD.add(**payload) + + return rt.to_dict() diff --git a/cmdb-api/api/lib/common_setting/common_data.py b/cmdb-api/api/lib/common_setting/common_data.py new file mode 100644 index 00000000..00c73981 --- /dev/null +++ b/cmdb-api/api/lib/common_setting/common_data.py @@ -0,0 +1,46 @@ +from flask import abort + +from api.extensions import db +from api.lib.common_setting.resp_format import ErrFormat +from api.models.common_setting import CommonData + + +class CommonDataCRUD(object): + + @staticmethod + def get_data_by_type(data_type): + return CommonData.get_by(data_type=data_type) + + @staticmethod + def get_data_by_id(_id, to_dict=True): + return CommonData.get_by(first=True, id=_id, to_dict=to_dict) + + @staticmethod + def create_new_data(data_type, **kwargs): + try: + return CommonData.create(data_type=data_type, **kwargs) + except Exception as e: + db.session.rollback() + abort(400, str(e)) + + @staticmethod + def update_data(_id, **kwargs): + existed = CommonDataCRUD.get_data_by_id(_id, to_dict=False) + if not existed: + abort(404, ErrFormat.common_data_not_found.format(_id)) + try: + return existed.update(**kwargs) + except Exception as e: + db.session.rollback() + abort(400, str(e)) + + @staticmethod + def delete(_id): + existed = CommonDataCRUD.get_data_by_id(_id, to_dict=False) + if not existed: + abort(404, ErrFormat.common_data_not_found.format(_id)) + try: + existed.soft_delete() + except Exception as e: + db.session.rollback() + abort(400, str(e)) diff --git a/cmdb-api/api/lib/common_setting/company_info.py b/cmdb-api/api/lib/common_setting/company_info.py index 861bb3fe..c700fea1 100644 --- a/cmdb-api/api/lib/common_setting/company_info.py +++ b/cmdb-api/api/lib/common_setting/company_info.py @@ -1,5 +1,5 @@ # -*- coding:utf-8 -*- - +from api.extensions import cache from api.models.common_setting import CompanyInfo @@ -11,14 +11,34 @@ def get(): @staticmethod def create(**kwargs): - return CompanyInfo.create(**kwargs) + res = CompanyInfo.create(**kwargs) + CompanyInfoCache.refresh(res.info) + return res @staticmethod def update(_id, **kwargs): kwargs.pop('id', None) existed = CompanyInfo.get_by_id(_id) if not existed: - return CompanyInfoCRUD.create(**kwargs) + existed = CompanyInfoCRUD.create(**kwargs) else: existed = existed.update(**kwargs) - return existed + CompanyInfoCache.refresh(existed.info) + return existed + + +class CompanyInfoCache(object): + key = 'CompanyInfoCache::' + + @classmethod + def get(cls): + info = cache.get(cls.key) + if not info: + res = CompanyInfo.get_by(first=True) or {} + info = res.get('info', {}) + cache.set(cls.key, info) + return info + + @classmethod + def refresh(cls, info): + cache.set(cls.key, info) \ No newline at end of file diff --git a/cmdb-api/api/lib/common_setting/const.py b/cmdb-api/api/lib/common_setting/const.py index c9edccde..bb585d6c 100644 --- a/cmdb-api/api/lib/common_setting/const.py +++ b/cmdb-api/api/lib/common_setting/const.py @@ -4,11 +4,18 @@ class OperatorType(BaseEnum): - EQUAL = 1 # 等于 - NOT_EQUAL = 2 # 不等于 - IN = 3 # 包含 - NOT_IN = 4 # 不包含 - GREATER_THAN = 5 # 大于 - LESS_THAN = 6 # 小于 - IS_EMPTY = 7 # 为空 - IS_NOT_EMPTY = 8 # 不为空 + EQUAL = 1 + NOT_EQUAL = 2 + IN = 3 + NOT_IN = 4 + GREATER_THAN = 5 + LESS_THAN = 6 + IS_EMPTY = 7 + IS_NOT_EMPTY = 8 + + +BotNameMap = { + 'wechatApp': 'wechatBot', + 'feishuApp': 'feishuBot', + 'dingdingApp': 'dingdingBot', +} diff --git a/cmdb-api/api/lib/common_setting/department.py b/cmdb-api/api/lib/common_setting/department.py index 6115014e..f068b53a 100644 --- a/cmdb-api/api/lib/common_setting/department.py +++ b/cmdb-api/api/lib/common_setting/department.py @@ -1,47 +1,33 @@ # -*- coding:utf-8 -*- -from flask import abort +from flask import abort, current_app from treelib import Tree from wtforms import Form from wtforms import IntegerField from wtforms import StringField from wtforms import validators +from api.extensions import db from api.lib.common_setting.resp_format import ErrFormat -from api.lib.common_setting.utils import get_df_from_read_sql +from api.lib.common_setting.acl import ACLManager from api.lib.perm.acl.role import RoleCRUD from api.models.common_setting import Department, Employee sub_departments_column_name = 'sub_departments' -def drop_ts_column(df): - columns = list(df.columns) - remove_columns = [] - for column in ['created_at', 'updated_at', 'deleted_at', 'last_login']: - targets = list(filter(lambda c: c.startswith(column), columns)) - if targets: - remove_columns.extend(targets) - - remove_columns = list(set(remove_columns)) - - return df.drop(remove_columns, axis=1) if len(remove_columns) > 0 else df - - -def get_department_df(): +def get_all_department_list(to_dict=True): criterion = [ Department.deleted == 0, ] query = Department.query.filter( *criterion - ) - df = get_df_from_read_sql(query) - if df.empty: - return - return drop_ts_column(df) + ).order_by(Department.department_id.asc()) + results = query.all() + return [r.to_dict() for r in results] if to_dict else results -def get_all_employee_df(block=0): +def get_all_employee_list(block=0, to_dict=True): criterion = [ Employee.deleted == 0, ] @@ -50,112 +36,106 @@ def get_all_employee_df(block=0): Employee.block == block ) - entities = [getattr(Employee, c) for c in Employee.get_columns( - ).keys() if c not in ['deleted', 'deleted_at']] - query = Employee.query.with_entities( - *entities - ).filter( - *criterion - ) - df = get_df_from_read_sql(query) - if df.empty: - return df - return drop_ts_column(df) + results = db.session.query(Employee).filter(*criterion).all() + + DepartmentTreeEmployeeColumns = [ + 'acl_rid', + 'employee_id', + 'username', + 'nickname', + 'email', + 'mobile', + 'direct_supervisor_id', + 'block', + 'department_id', + ] + + def format_columns(e): + return {column: getattr(e, column) for column in DepartmentTreeEmployeeColumns} + + return [format_columns(r) for r in results] if to_dict else results class DepartmentTree(object): def __init__(self, append_employee=False, block=-1): self.append_employee = append_employee self.block = block - self.d_df = get_department_df() - self.employee_df = get_all_employee_df( + self.all_department_list = get_all_department_list() + self.all_employee_list = get_all_employee_list( block) if append_employee else None def prepare(self): pass def get_employees_by_d_id(self, d_id): - _df = self.employee_df[ - self.employee_df['department_id'].eq(d_id) - ].sort_values(by=['direct_supervisor_id'], ascending=True) - if _df.empty: - return [] + block = self.block - if self.block != -1: - _df = _df[ - _df['block'].eq(self.block) - ] + def filter_department_id(e): + if self.block != -1: + return e['department_id'] == d_id and e['block'] == block + return e.department_id == d_id - return _df.to_dict('records') + results = list(filter(lambda e: filter_department_id(e), self.all_employee_list)) + + return results + + def get_department_by_parent_id(self, parent_id): + results = list(filter(lambda d: d['department_parent_id'] == parent_id, self.all_department_list)) + if not results: + return [] + return results def get_tree_departments(self): # 一级部门 - top_df = self.d_df[self.d_df['department_parent_id'].eq(-1)] - if top_df.empty: + top_departments = self.get_department_by_parent_id(-1) + if len(top_departments) == 0: return [] d_list = [] - for index in top_df.index: - top_d = top_df.loc[index].to_dict() - + for top_d in top_departments: department_id = top_d['department_id'] - - # 检查 department_id 是否作为其他部门的 parent - sub_df = self.d_df[ - self.d_df['department_parent_id'].eq(department_id) - ].sort_values(by=['sort_value'], ascending=True) - + sub_deps = self.get_department_by_parent_id(department_id) employees = [] - if self.append_employee: - # 要包含员工 employees = self.get_employees_by_d_id(department_id) top_d['employees'] = employees - - if sub_df.empty: + if len(sub_deps) == 0: top_d[sub_departments_column_name] = [] d_list.append(top_d) continue - self.parse_sub_department(sub_df, top_d) + self.parse_sub_department(sub_deps, top_d) d_list.append(top_d) return d_list def get_all_departments(self, is_tree=1): - if self.d_df.empty: + if len(self.all_department_list) == 0: return [] if is_tree != 1: - return self.d_df.to_dict('records') + return self.all_department_list return self.get_tree_departments() - def parse_sub_department(self, df, top_d): + def parse_sub_department(self, deps, top_d): sub_departments = [] - for s_index in df.index: - d = df.loc[s_index].to_dict() - sub_df = self.d_df[ - self.d_df['department_parent_id'].eq( - df.at[s_index, 'department_id']) - ].sort_values(by=['sort_value'], ascending=True) + for d in deps: + sub_deps = self.get_department_by_parent_id(d['department_id']) employees = [] - if self.append_employee: - # 要包含员工 - employees = self.get_employees_by_d_id( - df.at[s_index, 'department_id']) + employees = self.get_employees_by_d_id(d['department_id']) d['employees'] = employees - if sub_df.empty: + if len(sub_deps) == 0: d[sub_departments_column_name] = [] sub_departments.append(d) continue - self.parse_sub_department(sub_df, d) + self.parse_sub_department(sub_deps, d) sub_departments.append(d) top_d[sub_departments_column_name] = sub_departments @@ -173,6 +153,10 @@ class DepartmentForm(Form): class DepartmentCRUD(object): + @staticmethod + def get_department_by_id(d_id, to_dict=True): + return Department.get_by(first=True, department_id=d_id, to_dict=to_dict) + @staticmethod def add(**kwargs): DepartmentCRUD.check_department_name_unique(kwargs['department_name']) @@ -202,16 +186,16 @@ def add(**kwargs): def check_department_parent_id_allow(d_id, department_parent_id): if department_parent_id == 0: return - # 检查 department_parent_id 是否在许可范围内 allow_p_d_id_list = DepartmentCRUD.get_allow_parent_d_id_by(d_id) target = list( filter(lambda d: d['department_id'] == department_parent_id, allow_p_d_id_list)) if len(target) == 0: try: - d = Department.get_by( + dep = Department.get_by( first=True, to_dict=False, department_id=department_parent_id) - name = d.department_name if d else ErrFormat.department_id_not_found.format(department_parent_id) + name = dep.department_name if dep else ErrFormat.department_id_not_found.format(department_parent_id) except Exception as e: + current_app.logger.error(str(e)) name = ErrFormat.department_id_not_found.format(department_parent_id) abort(400, ErrFormat.cannot_to_be_parent_department.format(name)) @@ -275,15 +259,12 @@ def delete(_id): try: RoleCRUD.delete_role(existed.acl_rid) except Exception as e: - pass + current_app.logger.error(str(e)) return existed.soft_delete() @staticmethod def get_allow_parent_d_id_by(department_id): - """ - 获取可以成为 department_id 的 department_parent_id 的 list - """ tree_list = DepartmentCRUD.get_department_tree_list() allow_d_id_list = [] @@ -293,7 +274,7 @@ def get_allow_parent_d_id_by(department_id): try: tree.remove_subtree(department_id) except Exception as e: - pass + current_app.logger.error(str(e)) [allow_d_id_list.append({'department_id': int(n.identifier), 'department_name': n.tag}) for n in tree.all_nodes()] @@ -321,58 +302,57 @@ def get_all_departments_with_employee(block): @staticmethod def get_department_tree_list(): - df = get_department_df() - if df.empty: + all_deps = get_all_department_list() + if len(all_deps) == 0: return [] - # 一级部门 - top_df = df[df['department_parent_id'].eq(-1)] - if top_df.empty: + top_deps = list(filter(lambda d: d['department_parent_id'] == -1, all_deps)) + if len(top_deps) == 0: return [] tree_list = [] - for index in top_df.index: + for top_d in top_deps: tree = Tree() - identifier_root = top_df.at[index, 'department_id'] + identifier_root = top_d['department_id'] tree.create_node( - top_df.at[index, 'department_name'], + top_d['department_name'], identifier_root ) - - # 检查 department_id 是否作为其他部门的 parent - sub_df = df[ - df['department_parent_id'].eq(identifier_root) - ] - if sub_df.empty: + sub_ds = list(filter(lambda d: d['department_parent_id'] == identifier_root, all_deps)) + if len(sub_ds) == 0: tree_list.append(tree) continue DepartmentCRUD.parse_sub_department_node( - sub_df, df, tree, identifier_root) + sub_ds, all_deps, tree, identifier_root) tree_list.append(tree) return tree_list @staticmethod - def parse_sub_department_node(df, all_df, tree, parent_id): - for s_index in df.index: + def parse_sub_department_node(sub_ds, all_ds, tree, parent_id): + for d in sub_ds: tree.create_node( - df.at[s_index, 'department_name'], - df.at[s_index, 'department_id'], + d['department_name'], + d['department_id'], parent=parent_id ) - sub_df = all_df[ - all_df['department_parent_id'].eq( - df.at[s_index, 'department_id']) - ] - if sub_df.empty: + next_sub_ds = list(filter(lambda item_d: item_d['department_parent_id'] == d['department_id'], all_ds)) + if len(next_sub_ds) == 0: continue DepartmentCRUD.parse_sub_department_node( - sub_df, all_df, tree, df.at[s_index, 'department_id']) + next_sub_ds, all_ds, tree, d['department_id']) + + @staticmethod + def get_department_by_query(query, to_dict=True): + results = query.all() + if not results: + return [] + return results if not to_dict else [r.to_dict() for r in results] @staticmethod def get_departments_and_ids(department_parent_id, block): @@ -380,44 +360,30 @@ def get_departments_and_ids(department_parent_id, block): Department.department_parent_id == department_parent_id, Department.deleted == 0, ).order_by(Department.sort_value.asc()) - df = get_df_from_read_sql(query) - if df.empty: + all_departments = DepartmentCRUD.get_department_by_query(query) + if len(all_departments) == 0: return [], [] tree_list = DepartmentCRUD.get_department_tree_list() - employee_df = get_all_employee_df(block) + all_employee_list = get_all_employee_list(block) - department_id_list = list(df['department_id'].values) + department_id_list = [d['department_id'] for d in all_departments] query = Department.query.filter( Department.department_parent_id.in_(department_id_list), Department.deleted == 0, ).order_by(Department.sort_value.asc()).group_by(Department.department_id) - sub_df = get_df_from_read_sql(query) - if sub_df.empty: - df['has_sub'] = 0 + sub_deps = DepartmentCRUD.get_department_by_query(query) - def handle_row_employee_count(row): - return len(employee_df[employee_df['department_id'] == row['department_id']]) + sub_map = {d['department_parent_id']: 1 for d in sub_deps} - df['employee_count'] = df.apply( - lambda row: handle_row_employee_count(row), axis=1) + for d in all_departments: + d['has_sub'] = sub_map.get(d['department_id'], 0) - else: - sub_map = {d['department_parent_id']: 1 for d in sub_df.to_dict('records')} + d_ids = DepartmentCRUD.get_department_id_list_by_root(d['department_id'], tree_list) - def handle_row(row): - d_ids = DepartmentCRUD.get_department_id_list_by_root( - row['department_id'], tree_list) - row['employee_count'] = len( - employee_df[employee_df['department_id'].isin(d_ids)]) + d['employee_count'] = len(list(filter(lambda e: e['department_id'] in d_ids, all_employee_list))) - row['has_sub'] = sub_map.get(row['department_id'], 0) - - return row - - df = df.apply(lambda row: handle_row(row), axis=1) - - return df.to_dict('records'), department_id_list + return all_departments, department_id_list @staticmethod def get_department_id_list_by_root(root_department_id, tree_list=None): @@ -430,6 +396,125 @@ def get_department_id_list_by_root(root_department_id, tree_list=None): [id_list.append(int(n.identifier)) for n in tmp_tree.all_nodes()] except Exception as e: - pass + current_app.logger.error(str(e)) return id_list + + +class EditDepartmentInACL(object): + + @staticmethod + def add_department_to_acl(department_id, op_uid): + db_department = DepartmentCRUD.get_department_by_id(department_id, to_dict=False) + if not db_department: + return + + from api.models.acl import Role + role = Role.get_by(first=True, name=db_department.department_name, app_id=None) + + acl = ACLManager('acl', str(op_uid)) + if role is None: + payload = { + 'app_id': 'acl', + 'name': db_department.department_name, + } + role = acl.create_role(payload) + + acl_rid = role.get('id') if role else 0 + + db_department.update( + acl_rid=acl_rid + ) + info = f"add_department_to_acl, acl_rid: {acl_rid}" + current_app.logger.info(info) + return info + + @staticmethod + def delete_department_from_acl(department_rids, op_uid): + acl = ACLManager('acl', str(op_uid)) + + result = [] + + for rid in department_rids: + try: + acl.delete_role(rid) + except Exception as e: + result.append(f"delete_department_in_acl, rid: {rid}, error: {e}") + continue + + return result + + @staticmethod + def edit_department_name_in_acl(d_rid: int, d_name: str, op_uid: int): + acl = ACLManager('acl', str(op_uid)) + payload = { + 'name': d_name + } + try: + acl.edit_role(d_rid, payload) + except Exception as e: + return f"edit_department_name_in_acl, rid: {d_rid}, error: {e}" + + return f"edit_department_name_in_acl, rid: {d_rid}, success" + + @staticmethod + def edit_employee_department_in_acl(e_list: list, new_d_id: int, op_uid: int): + result = [] + new_department = DepartmentCRUD.get_department_by_id(new_d_id, False) + if not new_department: + result.append(f"{new_d_id} new_department is None") + return result + + from api.models.acl import Role + new_role = Role.get_by(first=True, name=new_department.department_name, app_id=None) + new_d_rid_in_acl = new_role.get('id') if new_role else 0 + if new_d_rid_in_acl == 0: + return + + if new_d_rid_in_acl != new_department.acl_rid: + new_department.update( + acl_rid=new_d_rid_in_acl + ) + new_department_acl_rid = new_department.acl_rid if new_d_rid_in_acl == new_department.acl_rid else \ + new_d_rid_in_acl + + acl = ACLManager('acl', str(op_uid)) + for employee in e_list: + old_department = DepartmentCRUD.get_department_by_id(employee.get('department_id'), False) + if not old_department: + continue + employee_acl_rid = employee.get('e_acl_rid') + if employee_acl_rid == 0: + result.append(f"employee_acl_rid == 0") + continue + + old_role = Role.get_by(first=True, name=old_department.department_name, app_id=None) + old_d_rid_in_acl = old_role.get('id') if old_role else 0 + if old_d_rid_in_acl == 0: + return + if old_d_rid_in_acl != old_department.acl_rid: + old_department.update( + acl_rid=old_d_rid_in_acl + ) + d_acl_rid = old_department.acl_rid if old_d_rid_in_acl == old_department.acl_rid else old_d_rid_in_acl + payload = { + 'app_id': 'acl', + 'parent_id': d_acl_rid, + } + try: + acl.remove_user_from_role(employee_acl_rid, payload) + except Exception as e: + result.append( + f"remove_user_from_role employee_acl_rid: {employee_acl_rid}, parent_id: {d_acl_rid}, err: {e}") + + payload = { + 'app_id': 'acl', + 'child_ids': [employee_acl_rid], + } + try: + acl.add_user_to_role(new_department_acl_rid, payload) + except Exception as e: + result.append( + f"add_user_to_role employee_acl_rid: {employee_acl_rid}, parent_id: {d_acl_rid}, err: {e}") + + return result diff --git a/cmdb-api/api/lib/common_setting/employee.py b/cmdb-api/api/lib/common_setting/employee.py index cf150ca5..cb75d4c5 100644 --- a/cmdb-api/api/lib/common_setting/employee.py +++ b/cmdb-api/api/lib/common_setting/employee.py @@ -1,9 +1,9 @@ # -*- coding:utf-8 -*- - +import copy import traceback from datetime import datetime -import pandas as pd +import requests from flask import abort from flask_login import current_user from sqlalchemy import or_, literal_column, func, not_, and_ @@ -17,9 +17,20 @@ from api.lib.common_setting.acl import ACLManager from api.lib.common_setting.const import COMMON_SETTING_QUEUE, OperatorType from api.lib.common_setting.resp_format import ErrFormat -from api.lib.common_setting.utils import get_df_from_read_sql from api.models.common_setting import Employee, Department +acl_user_columns = [ + 'email', + 'mobile', + 'nickname', + 'username', + 'password', + 'block', + 'avatar', +] +employee_pop_columns = ['password'] +can_not_edit_columns = ['email'] + def edit_acl_user(uid, **kwargs): user_data = {column: kwargs.get( @@ -70,9 +81,6 @@ def get_employee_by_id(_id): @staticmethod def get_employee_by_uid_with_create(_uid): - """ - 根据 uid 获取员工信息,不存在则创建 - """ try: return EmployeeCRUD.get_employee_by_uid(_uid).to_dict() except Exception as e: @@ -102,7 +110,6 @@ def check_acl_user_and_create(user_info): acl_uid=user_info['uid'], ) return existed.to_dict() - # 创建员工 if not user_info.get('nickname', None): user_info['nickname'] = user_info['name'] @@ -114,6 +121,19 @@ def check_acl_user_and_create(user_info): employee = CreateEmployee().create_single(**data) return employee.to_dict() + @staticmethod + def add_employee_from_acl_created(**kwargs): + try: + kwargs['acl_uid'] = kwargs.pop('uid') + kwargs['acl_rid'] = kwargs.pop('rid') + kwargs['department_id'] = 0 + + Employee.create( + **kwargs + ) + except Exception as e: + abort(400, str(e)) + @staticmethod def add(**kwargs): try: @@ -145,9 +165,6 @@ def update(_id, **kwargs): if len(e_list) > 0: from api.tasks.common_setting import edit_employee_department_in_acl - # fixme: comment next line - # edit_employee_department_in_acl(e_list, new_department_id, current_user.uid) - edit_employee_department_in_acl.apply_async( args=(e_list, new_department_id, current_user.uid), queue=COMMON_SETTING_QUEUE @@ -161,7 +178,7 @@ def update(_id, **kwargs): def edit_employee_by_uid(_uid, **kwargs): existed = EmployeeCRUD.get_employee_by_uid(_uid) try: - user = edit_acl_user(_uid, **kwargs) + edit_acl_user(_uid, **kwargs) for column in employee_pop_columns: if kwargs.get(column): @@ -173,9 +190,9 @@ def edit_employee_by_uid(_uid, **kwargs): @staticmethod def change_password_by_uid(_uid, password): - existed = EmployeeCRUD.get_employee_by_uid(_uid) + EmployeeCRUD.get_employee_by_uid(_uid) try: - user = edit_acl_user(_uid, password=password) + edit_acl_user(_uid, password=password) except Exception as e: return abort(400, str(e)) @@ -209,173 +226,6 @@ def get_employee_count(block_status): *criterion ).count() - @staticmethod - def import_employee(employee_list): - return CreateEmployee().batch_create(employee_list) - - @staticmethod - def get_export_employee_df(block_status): - criterion = [ - Employee.deleted == 0 - ] - if block_status >= 0: - criterion.append( - Employee.block == block_status - ) - - query = Employee.query.with_entities( - Employee.employee_id, - Employee.nickname, - Employee.email, - Employee.sex, - Employee.mobile, - Employee.position_name, - Employee.last_login, - Employee.department_id, - Employee.direct_supervisor_id, - ).filter(*criterion) - df = get_df_from_read_sql(query) - if df.empty: - return df - - query = Department.query.filter( - *criterion - ) - department_df = get_df_from_read_sql(query) - - def find_name(row): - department_id = row['department_id'] - _df = department_df[department_df['department_id'] - == department_id] - row['department_name'] = '' if _df.empty else _df.iloc[0]['department_name'] - - direct_supervisor_id = row['direct_supervisor_id'] - _df = df[df['employee_id'] == direct_supervisor_id] - row['nickname_direct_supervisor'] = '' if _df.empty else _df.iloc[0]['nickname'] - - if isinstance(row['last_login'], pd.Timestamp): - try: - row['last_login'] = str(row['last_login']) - except: - row['last_login'] = '' - else: - row['last_login'] = '' - - return row - - df = df.apply(find_name, axis=1) - df.drop(['department_id', 'direct_supervisor_id', - 'employee_id'], axis=1, inplace=True) - return df - - @staticmethod - def batch_employee(column_name, column_value, employee_id_list): - if not column_value: - abort(400, ErrFormat.value_is_required) - if column_name in ['password', 'block']: - return EmployeeCRUD.batch_edit_password_or_block_column(column_name, employee_id_list, column_value, True) - - elif column_name in ['department_id']: - return EmployeeCRUD.batch_edit_employee_department(employee_id_list, column_value) - - elif column_name in [ - 'direct_supervisor_id', 'position_name' - ]: - return EmployeeCRUD.batch_edit_column(column_name, employee_id_list, column_value, False) - - else: - abort(400, ErrFormat.column_name_not_support) - - @staticmethod - def batch_edit_employee_department(employee_id_list, column_value): - err_list = [] - employee_list = [] - for _id in employee_id_list: - try: - existed = EmployeeCRUD.get_employee_by_id(_id) - employee = dict( - e_acl_rid=existed.acl_rid, - department_id=existed.department_id - ) - employee_list.append(employee) - existed.update(department_id=column_value) - - except Exception as e: - err_list.append({ - 'employee_id': _id, - 'err': str(e), - }) - from api.tasks.common_setting import edit_employee_department_in_acl - edit_employee_department_in_acl.apply_async( - args=(employee_list, column_value, current_user.uid), - queue=COMMON_SETTING_QUEUE - ) - return err_list - - @staticmethod - def batch_edit_password_or_block_column(column_name, employee_id_list, column_value, is_acl=False): - if column_name == 'block': - err_list = [] - success_list = [] - for _id in employee_id_list: - try: - employee = EmployeeCRUD.edit_employee_block_column( - _id, is_acl, **{column_name: column_value}) - success_list.append(employee) - except Exception as e: - err_list.append({ - 'employee_id': _id, - 'err': str(e), - }) - return err_list - else: - return EmployeeCRUD.batch_edit_column(column_name, employee_id_list, column_value, is_acl) - - @staticmethod - def batch_edit_column(column_name, employee_id_list, column_value, is_acl=False): - err_list = [] - for _id in employee_id_list: - try: - EmployeeCRUD.edit_employee_single_column( - _id, is_acl, **{column_name: column_value}) - except Exception as e: - err_list.append({ - 'employee_id': _id, - 'err': str(e), - }) - - return err_list - - @staticmethod - def edit_employee_single_column(_id, is_acl=False, **kwargs): - existed = EmployeeCRUD.get_employee_by_id(_id) - - if is_acl: - return edit_acl_user(existed.acl_uid, **kwargs) - - try: - for column in employee_pop_columns: - if kwargs.get(column): - kwargs.pop(column) - - return existed.update(**kwargs) - except Exception as e: - return abort(400, str(e)) - - @staticmethod - def edit_employee_block_column(_id, is_acl=False, **kwargs): - existed = EmployeeCRUD.get_employee_by_id(_id) - value = get_block_value(kwargs.get('block')) - if value is True: - # 判断该用户是否为 部门负责人,或者员工的直接上级 - check_department_director_id_or_direct_supervisor_id(_id) - - if is_acl: - kwargs['block'] = value - edit_acl_user(existed.acl_uid, **kwargs) - data = existed.to_dict() - return data - @staticmethod def check_email_unique(email, _id=0): criterion = [ @@ -395,7 +245,7 @@ def check_email_unique(email, _id=0): raise Exception(err) @staticmethod - def get_employee_list_by_body(department_id, block_status, search='', order='', conditions=[], page=1, + def get_employee_list_by_body(department_id, block_status, search='', order='', conditions=None, page=1, page_size=10): criterion = [ Employee.deleted == 0 @@ -461,7 +311,7 @@ def parse_condition_list_to_query(condition_list): @staticmethod def get_expr_by_condition(column, operator, value, relation): """ - 根据conditions返回expr: (and_list, or_list) + get expr: (and_list, or_list) """ attr = EmployeeCRUD.get_attr_by_column(column) # 根据operator生成条件表达式 @@ -481,7 +331,7 @@ def get_expr_by_condition(column, operator, value, relation): if value: abort(400, ErrFormat.query_column_none_keep_value_empty.format(column)) expr = [attr.is_(None)] - if column not in ["entry_date", "leave_date", "dfc_entry_date", "last_login"]: + if column not in ["last_login"]: expr += [attr == ''] expr = [or_(*expr)] elif operator == OperatorType.IS_NOT_EMPTY: @@ -495,7 +345,6 @@ def get_expr_by_condition(column, operator, value, relation): else: abort(400, ErrFormat.not_support_operator.format(operator)) - # 根据relation生成复合条件 if relation == "&": return expr, [] elif relation == "|": @@ -505,15 +354,16 @@ def get_expr_by_condition(column, operator, value, relation): @staticmethod def check_condition(column, operator, value, relation): - # 对于condition中column为空的,报错 if column is None or operator is None or relation is None: return abort(400, ErrFormat.conditions_field_missing) if value and column == "last_login": try: - value = datetime.strptime(value, "%Y-%m-%d %H:%M:%S") + return datetime.strptime(value, "%Y-%m-%d %H:%M:%S") except Exception as e: - abort(400, ErrFormat.datetime_format_error.format(column)) + err = f"{ErrFormat.datetime_format_error.format(column)}: {str(e)}" + abort(400, err) + return value @staticmethod def get_attr_by_column(column): @@ -534,7 +384,7 @@ def get_query_by_conditions(query, conditions): relation = condition.get("relation", None) value = condition.get("value", None) - EmployeeCRUD.check_condition(column, operator, value, relation) + value = EmployeeCRUD.check_condition(column, operator, value, relation) a, o = EmployeeCRUD.get_expr_by_condition( column, operator, value, relation) and_list += a @@ -640,6 +490,202 @@ def get_employees_by_department_id(department_id, block): return [r.to_dict() for r in results] + @staticmethod + def remove_bind_notice_by_uid(_platform, _uid): + existed = EmployeeCRUD.get_employee_by_uid(_uid) + employee_data = existed.to_dict() + + notice_info = employee_data.get('notice_info', {}) + notice_info = copy.deepcopy(notice_info) if notice_info else {} + + notice_info[_platform] = '' + + existed.update( + notice_info=notice_info + ) + return ErrFormat.notice_remove_bind_success + + @staticmethod + def bind_notice_by_uid(_platform, _uid): + existed = EmployeeCRUD.get_employee_by_uid(_uid) + mobile = existed.mobile + if not mobile or len(mobile) == 0: + abort(400, ErrFormat.notice_bind_err_with_empty_mobile) + + from api.lib.common_setting.notice_config import NoticeConfigCRUD + messenger = NoticeConfigCRUD.get_messenger_url() + if not messenger or len(messenger) == 0: + abort(400, ErrFormat.notice_please_config_messenger_first) + + url = f"{messenger}/v1/uid/getbyphone" + try: + payload = dict( + phone=mobile, + sender=_platform + ) + res = requests.post(url, json=payload) + result = res.json() + if res.status_code != 200: + raise Exception(result.get('msg', '')) + target_id = result.get('uid', '') + + employee_data = existed.to_dict() + + notice_info = employee_data.get('notice_info', {}) + notice_info = copy.deepcopy(notice_info) if notice_info else {} + + notice_info[_platform] = '' if not target_id else target_id + + existed.update( + notice_info=notice_info + ) + return ErrFormat.notice_bind_success + + except Exception as e: + return abort(400, ErrFormat.notice_bind_failed.format(str(e))) + + @staticmethod + def get_employee_notice_by_ids(employee_ids): + criterion = [ + Employee.employee_id.in_(employee_ids), + Employee.deleted == 0, + ] + direct_columns = ['email', 'mobile'] + employees = Employee.query.filter( + *criterion + ).all() + results = [] + for employee in employees: + d = employee.to_dict() + tmp = dict( + employee_id=employee.employee_id, + ) + for column in direct_columns: + tmp[column] = d.get(column, '') + notice_info = d.get('notice_info', {}) + tmp.update(**notice_info) + results.append(tmp) + return results + + @staticmethod + def import_employee(employee_list): + res = CreateEmployee().batch_create(employee_list) + return res + + @staticmethod + def batch_edit_employee_department(employee_id_list, column_value): + err_list = [] + employee_list = [] + for _id in employee_id_list: + try: + existed = EmployeeCRUD.get_employee_by_id(_id) + employee = dict( + e_acl_rid=existed.acl_rid, + department_id=existed.department_id + ) + employee_list.append(employee) + existed.update(department_id=column_value) + + except Exception as e: + err_list.append({ + 'employee_id': _id, + 'err': str(e), + }) + from api.lib.common_setting.department import EditDepartmentInACL + EditDepartmentInACL.edit_employee_department_in_acl( + employee_list, column_value, current_user.uid + ) + return err_list + + @staticmethod + def batch_edit_password_or_block_column(column_name, employee_id_list, column_value, is_acl=False): + if column_name == 'block': + err_list = [] + success_list = [] + for _id in employee_id_list: + try: + employee = EmployeeCRUD.edit_employee_block_column( + _id, is_acl, **{column_name: column_value}) + success_list.append(employee) + except Exception as e: + err_list.append({ + 'employee_id': _id, + 'err': str(e), + }) + return err_list + else: + return EmployeeCRUD.batch_edit_column(column_name, employee_id_list, column_value, is_acl) + + @staticmethod + def batch_edit_column(column_name, employee_id_list, column_value, is_acl=False): + err_list = [] + for _id in employee_id_list: + try: + EmployeeCRUD.edit_employee_single_column( + _id, is_acl, **{column_name: column_value}) + except Exception as e: + err_list.append({ + 'employee_id': _id, + 'err': str(e), + }) + + return err_list + + @staticmethod + def edit_employee_single_column(_id, is_acl=False, **kwargs): + existed = EmployeeCRUD.get_employee_by_id(_id) + if 'direct_supervisor_id' in kwargs.keys(): + if kwargs['direct_supervisor_id'] == existed.direct_supervisor_id: + raise Exception(ErrFormat.direct_supervisor_is_not_self) + + if is_acl: + return edit_acl_user(existed.acl_uid, **kwargs) + + try: + for column in employee_pop_columns: + if kwargs.get(column): + kwargs.pop(column) + + return existed.update(**kwargs) + except Exception as e: + return abort(400, str(e)) + + @staticmethod + def edit_employee_block_column(_id, is_acl=False, **kwargs): + existed = EmployeeCRUD.get_employee_by_id(_id) + value = get_block_value(kwargs.get('block')) + if value is True: + check_department_director_id_or_direct_supervisor_id(_id) + value = 1 + else: + value = 0 + + if is_acl: + kwargs['block'] = value + edit_acl_user(existed.acl_uid, **kwargs) + + existed.update(block=value) + data = existed.to_dict() + return data + + @staticmethod + def batch_employee(column_name, column_value, employee_id_list): + if column_value is None: + abort(400, ErrFormat.value_is_required) + if column_name in ['password', 'block']: + return EmployeeCRUD.batch_edit_password_or_block_column(column_name, employee_id_list, column_value, True) + + elif column_name in ['department_id']: + return EmployeeCRUD.batch_edit_employee_department(employee_id_list, column_value) + + elif column_name in [ + 'direct_supervisor_id', 'position_name' + ]: + return EmployeeCRUD.batch_edit_column(column_name, employee_id_list, column_value, False) + + else: + abort(400, ErrFormat.column_name_not_support) + def get_user_map(key='uid', acl=None): """ @@ -654,19 +700,6 @@ def get_user_map(key='uid', acl=None): return data -acl_user_columns = [ - 'email', - 'mobile', - 'nickname', - 'username', - 'password', - 'block', - 'avatar', -] -employee_pop_columns = ['password'] -can_not_edit_columns = ['email'] - - def format_params(params): for k in ['_key', '_secret']: params.pop(k, None) @@ -676,19 +709,22 @@ def format_params(params): class CreateEmployee(object): def __init__(self): self.acl = ACLManager() - self.useremail_map = {} + self.all_acl_users = self.acl.get_all_users() - def check_acl_user(self, email): - user_info = self.useremail_map.get(email, None) - if user_info: - return user_info - return None + def check_acl_user(self, user_data): + target_email = list(filter(lambda x: x['email'] == user_data['email'], self.all_acl_users)) + if target_email: + return target_email[0] + + target_username = list(filter(lambda x: x['username'] == user_data['username'], self.all_acl_users)) + if target_username: + return target_username[0] def add_acl_user(self, **kwargs): user_data = {column: kwargs.get( column, '') for column in acl_user_columns if kwargs.get(column, '')} try: - existed = self.check_acl_user(user_data['email']) + existed = self.check_acl_user(user_data) if not existed: return self.acl.create_user(user_data) return existed @@ -697,8 +733,6 @@ def add_acl_user(self, **kwargs): def create_single(self, **kwargs): EmployeeCRUD.check_email_unique(kwargs['email']) - self.useremail_map = self.useremail_map if self.useremail_map else get_user_map( - 'email', self.acl) user = self.add_acl_user(**kwargs) kwargs['acl_uid'] = user['uid'] kwargs['last_login'] = user['last_login'] @@ -711,8 +745,6 @@ def create_single(self, **kwargs): ) def create_single_with_import(self, **kwargs): - self.useremail_map = self.useremail_map if self.useremail_map else get_user_map( - 'email', self.acl) user = self.add_acl_user(**kwargs) kwargs['acl_uid'] = user['uid'] kwargs['last_login'] = user['last_login'] @@ -730,7 +762,8 @@ def create_single_with_import(self, **kwargs): **kwargs ) - def get_department_by_name(self, d_name): + @staticmethod + def get_department_by_name(d_name): return Department.get_by(first=True, department_name=d_name) def get_end_department_id(self, department_name_list, department_name_map): @@ -755,9 +788,6 @@ def get_end_department_id(self, department_name_list, department_name_map): return end_d_id def format_department_id(self, employee): - """ - 部门名称转化为ID,不存在则创建 - """ department_name_map = {} try: department_name = employee.get('department_name', '') @@ -774,16 +804,13 @@ def format_department_id(self, employee): def batch_create(self, employee_list): err_list = [] - self.useremail_map = get_user_map('email', self.acl) for employee in employee_list: try: - # 获取username username = employee.get('username', None) if username is None: employee['username'] = employee['email'] - # 校验通过后获取department_id employee = self.format_department_id(employee) err = employee.get('err', None) if err: @@ -795,7 +822,7 @@ def batch_create(self, employee_list): raise Exception( ','.join(['{}: {}'.format(filed, ','.join(msg)) for filed, msg in form.errors.items()])) - data = self.create_single_with_import(**form.data) + self.create_single_with_import(**form.data) except Exception as e: err_list.append({ 'email': employee.get('email', ''), @@ -809,12 +836,12 @@ def batch_create(self, employee_list): class EmployeeAddForm(Form): username = StringField(validators=[ - validators.DataRequired(message="username不能为空"), + validators.DataRequired(message=ErrFormat.username_is_required), validators.Length(max=255), ]) email = StringField(validators=[ - validators.DataRequired(message="邮箱不能为空"), - validators.Email(message="邮箱格式不正确"), + validators.DataRequired(message=ErrFormat.email_is_required), + validators.Email(message=ErrFormat.email_format_error), validators.Length(max=255), ]) password = StringField(validators=[ @@ -823,7 +850,7 @@ class EmployeeAddForm(Form): position_name = StringField(validators=[]) nickname = StringField(validators=[ - validators.DataRequired(message="用户名不能为空"), + validators.DataRequired(message=ErrFormat.nickname_is_required), validators.Length(max=255), ]) sex = StringField(validators=[]) @@ -834,7 +861,7 @@ class EmployeeAddForm(Form): class EmployeeUpdateByUidForm(Form): nickname = StringField(validators=[ - validators.DataRequired(message="用户名不能为空"), + validators.DataRequired(message=ErrFormat.nickname_is_required), validators.Length(max=255), ]) avatar = StringField(validators=[]) diff --git a/cmdb-api/api/lib/common_setting/notice_config.py b/cmdb-api/api/lib/common_setting/notice_config.py new file mode 100644 index 00000000..152eb53d --- /dev/null +++ b/cmdb-api/api/lib/common_setting/notice_config.py @@ -0,0 +1,165 @@ +import requests + +from api.lib.common_setting.const import BotNameMap +from api.lib.common_setting.resp_format import ErrFormat +from api.models.common_setting import CompanyInfo, NoticeConfig +from wtforms import Form +from wtforms import StringField +from wtforms import validators +from flask import abort, current_app + + +class NoticeConfigCRUD(object): + + @staticmethod + def add_notice_config(**kwargs): + platform = kwargs.get('platform') + NoticeConfigCRUD.check_platform(platform) + info = kwargs.get('info', {}) + if 'name' not in info: + info['name'] = platform + kwargs['info'] = info + try: + NoticeConfigCRUD.update_messenger_config(**info) + res = NoticeConfig.create( + **kwargs + ) + return res + + except Exception as e: + return abort(400, str(e)) + + @staticmethod + def check_platform(platform): + NoticeConfig.get_by(first=True, to_dict=False, platform=platform) and \ + abort(400, ErrFormat.notice_platform_existed.format(platform)) + + @staticmethod + def edit_notice_config(_id, **kwargs): + existed = NoticeConfigCRUD.get_notice_config_by_id(_id) + try: + info = kwargs.get('info', {}) + if 'name' not in info: + info['name'] = existed.platform + kwargs['info'] = info + NoticeConfigCRUD.update_messenger_config(**info) + + res = existed.update(**kwargs) + return res + except Exception as e: + return abort(400, str(e)) + + @staticmethod + def get_messenger_url(): + from api.lib.common_setting.company_info import CompanyInfoCache + com_info = CompanyInfoCache.get() + if not com_info: + return + messenger = com_info.get('messenger', '') + if len(messenger) == 0: + return + if messenger[-1] == '/': + messenger = messenger[:-1] + return messenger + + @staticmethod + def update_messenger_config(**kwargs): + try: + messenger = NoticeConfigCRUD.get_messenger_url() + if not messenger or len(messenger) == 0: + raise Exception(ErrFormat.notice_please_config_messenger_first) + + url = f"{messenger}/v1/senders" + name = kwargs.get('name') + bot_list = kwargs.pop('bot', None) + for k, v in kwargs.items(): + if isinstance(v, bool): + kwargs[k] = 'true' if v else 'false' + else: + kwargs[k] = str(v) + + payload = {name: [kwargs]} + current_app.logger.info(f"update_messenger_config: {url}, {payload}") + res = requests.put(url, json=payload, timeout=2) + current_app.logger.info(f"update_messenger_config: {res.status_code}, {res.text}") + + if not bot_list or len(bot_list) == 0: + return + bot_name = BotNameMap.get(name) + payload = {bot_name: bot_list} + current_app.logger.info(f"update_messenger_config: {url}, {payload}") + bot_res = requests.put(url, json=payload, timeout=2) + current_app.logger.info(f"update_messenger_config: {bot_res.status_code}, {bot_res.text}") + + except Exception as e: + return abort(400, str(e)) + + @staticmethod + def get_notice_config_by_id(_id): + return NoticeConfig.get_by(first=True, to_dict=False, id=_id) or \ + abort(400, + ErrFormat.notice_not_existed.format(_id)) + + @staticmethod + def get_all(): + return NoticeConfig.get_by(to_dict=True) + + @staticmethod + def test_send_email(receive_address, **kwargs): + messenger = NoticeConfigCRUD.get_messenger_url() + if not messenger or len(messenger) == 0: + abort(400, ErrFormat.notice_please_config_messenger_first) + url = f"{messenger}/v1/message" + + recipient_email = receive_address + + subject = 'Test Email' + body = 'This is a test email' + payload = { + "sender": 'email', + "msgtype": "text/plain", + "title": subject, + "content": body, + "tos": [recipient_email], + } + current_app.logger.info(f"test_send_email: {url}, {payload}") + response = requests.post(url, json=payload) + if response.status_code != 200: + abort(400, response.text) + + return 1 + + @staticmethod + def get_app_bot(): + result = [] + for notice_app in NoticeConfig.get_by(to_dict=False): + if notice_app.platform in ['email']: + continue + info = notice_app.info + name = info.get('name', '') + if name not in BotNameMap: + continue + result.append(dict( + name=info.get('name', ''), + label=info.get('label', ''), + bot=info.get('bot', []), + )) + return result + + +class NoticeConfigForm(Form): + platform = StringField(validators=[ + validators.DataRequired(message="平台 不能为空"), + validators.Length(max=255), + ]) + info = StringField(validators=[ + validators.DataRequired(message="信息 不能为空"), + validators.Length(max=255), + ]) + + +class NoticeConfigUpdateForm(Form): + info = StringField(validators=[ + validators.DataRequired(message="信息 不能为空"), + validators.Length(max=255), + ]) diff --git a/cmdb-api/api/lib/common_setting/resp_format.py b/cmdb-api/api/lib/common_setting/resp_format.py index 88f5284c..4c2d6f7f 100644 --- a/cmdb-api/api/lib/common_setting/resp_format.py +++ b/cmdb-api/api/lib/common_setting/resp_format.py @@ -49,3 +49,17 @@ class ErrFormat(CommonErrFormat): acl_add_user_to_role_failed = "ACL 添加用户到角色失败: {}" acl_import_user_failed = "ACL 导入用户[{}]失败: {}" + nickname_is_required = "用户名不能为空" + username_is_required = "username不能为空" + email_is_required = "邮箱不能为空" + email_format_error = "邮箱格式错误" + email_send_timeout = "邮件发送超时" + + common_data_not_found = "ID {} 找不到记录" + notice_platform_existed = "{} 已存在" + notice_not_existed = "{} 配置项不存在" + notice_please_config_messenger_first = "请先配置 messenger" + notice_bind_err_with_empty_mobile = "绑定失败,手机号为空" + notice_bind_failed = "绑定失败: {}" + notice_bind_success = "绑定成功" + notice_remove_bind_success = "解绑成功" diff --git a/cmdb-api/api/lib/common_setting/upload_file.py b/cmdb-api/api/lib/common_setting/upload_file.py index d7eca290..85fe697a 100644 --- a/cmdb-api/api/lib/common_setting/upload_file.py +++ b/cmdb-api/api/lib/common_setting/upload_file.py @@ -4,8 +4,7 @@ def allowed_file(filename, allowed_extensions): - return '.' in filename and \ - filename.rsplit('.', 1)[1].lower() in allowed_extensions + return '.' in filename and filename.rsplit('.', 1)[1].lower() in allowed_extensions def generate_new_file_name(name): @@ -13,4 +12,5 @@ def generate_new_file_name(name): prev_name = ''.join(name.split(f".{ext}")[:-1]) uid = str(uuid.uuid4()) cur_str = get_cur_time_str('_') + return f"{prev_name}_{cur_str}_{uid}.{ext}" diff --git a/cmdb-api/api/lib/common_setting/utils.py b/cmdb-api/api/lib/common_setting/utils.py index 4446a05a..138a3155 100644 --- a/cmdb-api/api/lib/common_setting/utils.py +++ b/cmdb-api/api/lib/common_setting/utils.py @@ -1,23 +1,6 @@ # -*- coding:utf-8 -*- from datetime import datetime -import pandas as pd -from sqlalchemy import text - -from api.extensions import db - - -def get_df_from_read_sql(query, to_dict=False): - bind = query.session.bind - query = query.statement.compile(dialect=bind.dialect if bind else None, - compile_kwargs={"literal_binds": True}).string - a = db.engine - df = pd.read_sql(sql=text(query), con=a.connect()) - - if to_dict: - return df.to_dict('records') - return df - def get_cur_time_str(split_flag='-'): f = f"%Y{split_flag}%m{split_flag}%d{split_flag}%H{split_flag}%M{split_flag}%S{split_flag}%f" diff --git a/cmdb-api/api/lib/database.py b/cmdb-api/api/lib/database.py index 5145bd49..d991d1a3 100644 --- a/cmdb-api/api/lib/database.py +++ b/cmdb-api/api/lib/database.py @@ -10,14 +10,18 @@ class FormatMixin(object): def to_dict(self): - res = dict([(k, getattr(self, k) if not isinstance( - getattr(self, k), (datetime.datetime, datetime.date, datetime.time)) else str( - getattr(self, k))) for k in getattr(self, "__mapper__").c.keys()]) - # FIXME: getattr(cls, "__table__").columns k.name + res = dict() + for k in getattr(self, "__mapper__").c.keys(): + if k in {'password', '_password', 'secret', '_secret'}: + continue - res.pop('password', None) - res.pop('_password', None) - res.pop('secret', None) + if k.startswith('_'): + k = k[1:] + + if not isinstance(getattr(self, k), (datetime.datetime, datetime.date, datetime.time)): + res[k] = getattr(self, k) + else: + res[k] = str(getattr(self, k)) return res @@ -80,10 +84,10 @@ def delete(self, flush=False, commit=True): db.session.rollback() raise CommitException(str(e)) - def soft_delete(self, flush=False): + def soft_delete(self, flush=False, commit=True): setattr(self, "deleted", True) setattr(self, "deleted_at", datetime.datetime.now()) - self.save(flush=flush) + self.save(flush=flush, commit=commit) @classmethod def get_by_id(cls, _id): @@ -138,8 +142,11 @@ def get_by(cls, first=False, return result[0] if first and result else (None if first else result) @classmethod - def get_by_like(cls, to_dict=True, **kwargs): + def get_by_like(cls, to_dict=True, deleted=False, **kwargs): query = db.session.query(cls) + if hasattr(cls, "deleted") and deleted is not None: + query = query.filter(cls.deleted.is_(deleted)) + for k, v in kwargs.items(): query = query.filter(getattr(cls, k).ilike('%{0}%'.format(v))) return [i.to_dict() if to_dict else i for i in query] diff --git a/cmdb-api/api/lib/decorator.py b/cmdb-api/api/lib/decorator.py index 94b0ce58..26247e6a 100644 --- a/cmdb-api/api/lib/decorator.py +++ b/cmdb-api/api/lib/decorator.py @@ -4,8 +4,14 @@ from functools import wraps from flask import abort +from flask import current_app from flask import request +from sqlalchemy.exc import InvalidRequestError +from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import PendingRollbackError +from sqlalchemy.exc import StatementError +from api.extensions import db from api.lib.resp_format import CommonErrFormat @@ -55,8 +61,8 @@ def wrapper(*args, **kwargs): if exclude_args and arg in exclude_args: continue - if attr.type.python_type == str and attr.type.length and \ - len(request.values[arg] or '') > attr.type.length: + if attr.type.python_type == str and attr.type.length and ( + len(request.values[arg] or '') > attr.type.length): return abort(400, CommonErrFormat.argument_str_length_limit.format(arg, attr.type.length)) elif attr.type.python_type in (int, float) and request.values[arg]: @@ -70,3 +76,43 @@ def wrapper(*args, **kwargs): return wrapper return decorate + + +def reconnect_db(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except (StatementError, OperationalError, InvalidRequestError) as e: + error_msg = str(e) + if 'Lost connection' in error_msg or 'reconnect until invalid transaction' in error_msg or \ + 'can be emitted within this transaction' in error_msg: + current_app.logger.info('[reconnect_db] lost connect rollback then retry') + db.session.rollback() + return func(*args, **kwargs) + else: + raise e + except Exception as e: + raise e + + return wrapper + + +def _flush_db(): + try: + db.session.commit() + except (StatementError, OperationalError, InvalidRequestError, PendingRollbackError): + db.session.rollback() + + +def flush_db(func): + @wraps(func) + def wrapper(*args, **kwargs): + _flush_db() + return func(*args, **kwargs) + + return wrapper + + +def run_flush_db(): + _flush_db() diff --git a/cmdb-api/api/lib/http_cli.py b/cmdb-api/api/lib/http_cli.py index 6c59777c..9fcdd280 100644 --- a/cmdb-api/api/lib/http_cli.py +++ b/cmdb-api/api/lib/http_cli.py @@ -4,21 +4,22 @@ import hashlib import requests -from future.moves.urllib.parse import urlparse from flask import abort -from flask import g from flask import current_app +from flask_login import current_user +from future.moves.urllib.parse import urlparse def build_api_key(path, params): - g.user is not None or abort(403, u"您得登陆才能进行该操作") - key = g.user.key - secret = g.user.secret + current_user is not None or abort(403, u"您得登陆才能进行该操作") + key = current_user.key + secret = current_user.secret values = "".join([str(params[k]) for k in sorted(params.keys()) if params[k] is not None]) if params.keys() else "" _secret = "".join([path, secret, values]).encode("utf-8") params["_secret"] = hashlib.sha1(_secret).hexdigest() params["_key"] = key + return params diff --git a/cmdb-api/api/lib/notify.py b/cmdb-api/api/lib/notify.py new file mode 100644 index 00000000..8d6e0701 --- /dev/null +++ b/cmdb-api/api/lib/notify.py @@ -0,0 +1,72 @@ +# -*- coding:utf-8 -*- + +import json + +import requests +import six +from flask import current_app +from jinja2 import Template +from markdownify import markdownify as md + +from api.lib.common_setting.notice_config import NoticeConfigCRUD +from api.lib.mail import send_mail + + +def _request_messenger(subject, body, tos, sender, payload): + params = dict(sender=sender, title=subject, + tos=[to[sender] for to in tos if to.get(sender)]) + + if not params['tos']: + raise Exception("no receivers") + + flat_tos = [] + for i in params['tos']: + if i.strip(): + to = Template(i).render(payload) + if isinstance(to, list): + flat_tos.extend(to) + elif isinstance(to, six.string_types): + flat_tos.append(to) + params['tos'] = flat_tos + + if sender == "email": + params['msgtype'] = 'text/html' + params['content'] = body + else: + params['msgtype'] = 'markdown' + try: + content = md("{}\n{}".format(subject or '', body or '')) + except Exception as e: + current_app.logger.warning("html2markdown failed: {}".format(e)) + content = "{}\n{}".format(subject or '', body or '') + + params['content'] = json.dumps(dict(content=content)) + + url = current_app.config.get('MESSENGER_URL') or NoticeConfigCRUD.get_messenger_url() + if not url: + raise Exception("no messenger url") + + if not url.endswith("message"): + url = "{}/v1/message".format(url) + + resp = requests.post(url, json=params) + if resp.status_code != 200: + raise Exception(resp.text) + + return resp.text + + +def notify_send(subject, body, methods, tos, payload=None): + payload = payload or {} + payload = {k: v or '' for k, v in payload.items()} + subject = Template(subject).render(payload) + body = Template(body).render(payload) + + res = '' + for method in methods: + if method == "email" and not current_app.config.get('USE_MESSENGER', True): + send_mail(None, [Template(to.get('email')).render(payload) for to in tos], subject, body) + + res += (_request_messenger(subject, body, tos, method, payload) + "\n") + + return res diff --git a/cmdb-api/api/lib/perm/acl/acl.py b/cmdb-api/api/lib/perm/acl/acl.py index 66dc7a24..423d3bec 100644 --- a/cmdb-api/api/lib/perm/acl/acl.py +++ b/cmdb-api/api/lib/perm/acl/acl.py @@ -5,8 +5,11 @@ import requests import six -from flask import current_app, g, request -from flask import session, abort +from flask import abort +from flask import current_app +from flask import request +from flask import session +from flask_login import current_user from api.extensions import cache from api.lib.perm.acl.audit import AuditCRUD @@ -84,8 +87,8 @@ def _get_role(self, name): if user: return Role.get_by(name=name, uid=user.uid, first=True, to_dict=False) - return Role.get_by(name=name, app_id=self.app_id, first=True, to_dict=False) or \ - Role.get_by(name=name, first=True, to_dict=False) + return (Role.get_by(name=name, app_id=self.app_id, first=True, to_dict=False) or + Role.get_by(name=name, first=True, to_dict=False)) def add_resource(self, name, resource_type_name=None): resource_type = ResourceType.get_by(name=resource_type_name, first=True, to_dict=False) @@ -154,9 +157,9 @@ def has_permission(self, resource_name, resource_type, perm, resource_id=None): if is_app_admin(self.app_id): return True - role = self._get_role(g.user.username) + role = self._get_role(current_user.username) - role or abort(404, ErrFormat.role_not_found.format(g.user.username)) + role or abort(404, ErrFormat.role_not_found.format(current_user.username)) return RoleCRUD.has_permission(role.id, resource_name, resource_type, self.app_id, perm, resource_id=resource_id) @@ -193,9 +196,9 @@ def get_user_info(username, app_id=None): return user def get_resources(self, resource_type_name=None): - role = self._get_role(g.user.username) + role = self._get_role(current_user.username) - role or abort(404, ErrFormat.role_not_found.format(g.user.username)) + role or abort(404, ErrFormat.role_not_found.format(current_user.username)) rid = role.id return RoleCRUD.recursive_resources(rid, self.app_id, resource_type_name).get('resources') @@ -215,7 +218,7 @@ def validate_permission(resources, resource_type, perm, app=None): return if current_app.config.get("USE_ACL"): - if g.user.username == "worker": + if current_user.username == "worker": return resources = [resources] if isinstance(resources, six.string_types) else resources @@ -313,7 +316,7 @@ def wrapper_role_required(*args, **kwargs): return if current_app.config.get("USE_ACL"): - if getattr(g.user, 'username', None) == "worker": + if getattr(current_user, 'username', None) == "worker": return func(*args, **kwargs) if role_name not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin(app): diff --git a/cmdb-api/api/lib/perm/acl/app.py b/cmdb-api/api/lib/perm/acl/app.py index 93772bc0..cf364326 100644 --- a/cmdb-api/api/lib/perm/acl/app.py +++ b/cmdb-api/api/lib/perm/acl/app.py @@ -8,7 +8,9 @@ from flask import current_app from api.extensions import db -from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditScope +from api.lib.perm.acl.audit import AuditCRUD +from api.lib.perm.acl.audit import AuditOperateType +from api.lib.perm.acl.audit import AuditScope from api.lib.perm.acl.resp_format import ErrFormat from api.models.acl import App diff --git a/cmdb-api/api/lib/perm/acl/audit.py b/cmdb-api/api/lib/perm/acl/audit.py index 819c9d3e..4732b9c5 100644 --- a/cmdb-api/api/lib/perm/acl/audit.py +++ b/cmdb-api/api/lib/perm/acl/audit.py @@ -4,13 +4,21 @@ from enum import Enum from typing import List -from flask import g, has_request_context, request +from flask import has_request_context, request from flask_login import current_user from sqlalchemy import func from api.lib.perm.acl import AppCache -from api.models.acl import AuditRoleLog, AuditResourceLog, AuditPermissionLog, AuditTriggerLog, RolePermission, \ - Resource, ResourceGroup, Permission, Role, ResourceType +from api.models.acl import AuditPermissionLog +from api.models.acl import AuditResourceLog +from api.models.acl import AuditRoleLog +from api.models.acl import AuditTriggerLog +from api.models.acl import Permission +from api.models.acl import Resource +from api.models.acl import ResourceGroup +from api.models.acl import ResourceType +from api.models.acl import Role +from api.models.acl import RolePermission class AuditScope(str, Enum): @@ -49,9 +57,7 @@ class AuditCRUD(object): @staticmethod def get_current_operate_uid(uid=None): - - user_id = uid or (hasattr(g, 'user') and getattr(g.user, 'uid', None)) \ - or getattr(current_user, 'user_id', None) + user_id = uid or (getattr(current_user, 'uid', None)) or getattr(current_user, 'user_id', None) if has_request_context() and request.headers.get('X-User-Id'): _user_id = request.headers['X-User-Id'] @@ -93,11 +99,8 @@ def search_permission(app_id, q=None, page=1, page_size=10, start=None, end=None criterion.append(AuditPermissionLog.operate_type == v) records = AuditPermissionLog.query.filter( - AuditPermissionLog.deleted == 0, - *criterion) \ - .order_by(AuditPermissionLog.id.desc()) \ - .offset((page - 1) * page_size) \ - .limit(page_size).all() + AuditPermissionLog.deleted == 0, *criterion).order_by( + AuditPermissionLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all() data = { 'data': [r.to_dict() for r in records], @@ -160,10 +163,8 @@ def search_role(app_id, q=None, page=1, page_size=10, start=None, end=None): elif k == 'operate_type': criterion.append(AuditRoleLog.operate_type == v) - records = AuditRoleLog.query.filter(AuditRoleLog.deleted == 0, *criterion) \ - .order_by(AuditRoleLog.id.desc()) \ - .offset((page - 1) * page_size) \ - .limit(page_size).all() + records = AuditRoleLog.query.filter(AuditRoleLog.deleted == 0, *criterion).order_by( + AuditRoleLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all() data = { 'data': [r.to_dict() for r in records], @@ -225,11 +226,8 @@ def search_resource(app_id, q=None, page=1, page_size=10, start=None, end=None): criterion.append(AuditResourceLog.operate_type == v) records = AuditResourceLog.query.filter( - AuditResourceLog.deleted == 0, - *criterion) \ - .order_by(AuditResourceLog.id.desc()) \ - .offset((page - 1) * page_size) \ - .limit(page_size).all() + AuditResourceLog.deleted == 0, *criterion).order_by( + AuditResourceLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all() data = { 'data': [r.to_dict() for r in records], @@ -259,11 +257,8 @@ def search_trigger(app_id, q=None, page=1, page_size=10, start=None, end=None): criterion.append(AuditTriggerLog.operate_type == v) records = AuditTriggerLog.query.filter( - AuditTriggerLog.deleted == 0, - *criterion) \ - .order_by(AuditTriggerLog.id.desc()) \ - .offset((page - 1) * page_size) \ - .limit(page_size).all() + AuditTriggerLog.deleted == 0, *criterion).order_by( + AuditTriggerLog.id.desc()).offset((page - 1) * page_size).limit(page_size).all() data = { 'data': [r.to_dict() for r in records], diff --git a/cmdb-api/api/lib/perm/acl/cache.py b/cmdb-api/api/lib/perm/acl/cache.py index b4907593..7204dcab 100644 --- a/cmdb-api/api/lib/perm/acl/cache.py +++ b/cmdb-api/api/lib/perm/acl/cache.py @@ -4,7 +4,7 @@ import msgpack from api.extensions import cache -from api.extensions import db +from api.lib.decorator import flush_db from api.lib.utils import Lock from api.models.acl import App from api.models.acl import Permission @@ -60,15 +60,15 @@ class UserCache(object): @classmethod def get(cls, key): - user = cache.get(cls.PREFIX_ID.format(key)) or \ - cache.get(cls.PREFIX_NAME.format(key)) or \ - cache.get(cls.PREFIX_NICK.format(key)) or \ - cache.get(cls.PREFIX_WXID.format(key)) + user = (cache.get(cls.PREFIX_ID.format(key)) or + cache.get(cls.PREFIX_NAME.format(key)) or + cache.get(cls.PREFIX_NICK.format(key)) or + cache.get(cls.PREFIX_WXID.format(key))) if not user: - user = User.query.get(key) or \ - User.query.get_by_username(key) or \ - User.query.get_by_nickname(key) or \ - User.query.get_by_wxid(key) + user = (User.query.get(key) or + User.query.get_by_username(key) or + User.query.get_by_nickname(key) or + User.query.get_by_wxid(key)) if user: cls.set(user) @@ -221,9 +221,9 @@ def get_resources2(cls, rid, app_id): return msgpack.loads(r_g, raw=False) @classmethod + @flush_db def rebuild(cls, rid, app_id): cls.clean(rid, app_id) - db.session.remove() cls.get_parent_ids(rid, app_id) cls.get_child_ids(rid, app_id) @@ -235,9 +235,9 @@ def rebuild(cls, rid, app_id): cls.get_resources2(rid, app_id) @classmethod + @flush_db def rebuild2(cls, rid, app_id): cache.delete(cls.PREFIX_RESOURCES2.format(rid, app_id)) - db.session.remove() cls.get_resources2(rid, app_id) @classmethod diff --git a/cmdb-api/api/lib/perm/acl/permission.py b/cmdb-api/api/lib/perm/acl/permission.py index f0259cc7..0169ddd4 100644 --- a/cmdb-api/api/lib/perm/acl/permission.py +++ b/cmdb-api/api/lib/perm/acl/permission.py @@ -4,7 +4,9 @@ from flask import abort from api.extensions import db -from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditOperateSource +from api.lib.perm.acl.audit import AuditCRUD +from api.lib.perm.acl.audit import AuditOperateSource +from api.lib.perm.acl.audit import AuditOperateType from api.lib.perm.acl.cache import PermissionCache from api.lib.perm.acl.cache import RoleCache from api.lib.perm.acl.cache import UserCache @@ -97,8 +99,8 @@ def grant(rid, perms, resource_id=None, group_id=None, rebuild=True, source=Audi elif group_id is not None: from api.models.acl import ResourceGroup - group = ResourceGroup.get_by_id(group_id) or \ - abort(404, ErrFormat.resource_group_not_found.format("id={}".format(group_id))) + group = ResourceGroup.get_by_id(group_id) or abort( + 404, ErrFormat.resource_group_not_found.format("id={}".format(group_id))) app_id = group.app_id rt_id = group.resource_type_id if not perms: @@ -206,8 +208,8 @@ def revoke(rid, perms, resource_id=None, group_id=None, rebuild=True, source=Aud if resource_id is not None: from api.models.acl import Resource - resource = Resource.get_by_id(resource_id) or \ - abort(404, ErrFormat.resource_not_found.format("id={}".format(resource_id))) + resource = Resource.get_by_id(resource_id) or abort( + 404, ErrFormat.resource_not_found.format("id={}".format(resource_id))) app_id = resource.app_id rt_id = resource.resource_type_id if not perms: @@ -216,8 +218,8 @@ def revoke(rid, perms, resource_id=None, group_id=None, rebuild=True, source=Aud elif group_id is not None: from api.models.acl import ResourceGroup - group = ResourceGroup.get_by_id(group_id) or \ - abort(404, ErrFormat.resource_group_not_found.format("id={}".format(group_id))) + group = ResourceGroup.get_by_id(group_id) or abort( + 404, ErrFormat.resource_group_not_found.format("id={}".format(group_id))) app_id = group.app_id rt_id = group.resource_type_id diff --git a/cmdb-api/api/lib/perm/acl/resource.py b/cmdb-api/api/lib/perm/acl/resource.py index c37f6935..f5128d4d 100644 --- a/cmdb-api/api/lib/perm/acl/resource.py +++ b/cmdb-api/api/lib/perm/acl/resource.py @@ -5,7 +5,9 @@ from flask import current_app from api.extensions import db -from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditScope +from api.lib.perm.acl.audit import AuditCRUD +from api.lib.perm.acl.audit import AuditOperateType +from api.lib.perm.acl.audit import AuditScope from api.lib.perm.acl.cache import ResourceCache from api.lib.perm.acl.cache import ResourceGroupCache from api.lib.perm.acl.cache import UserCache @@ -102,8 +104,8 @@ def update(cls, rt_id, **kwargs): @classmethod def delete(cls, rt_id): - rt = ResourceType.get_by_id(rt_id) or \ - abort(404, ErrFormat.resource_type_not_found.format("id={}".format(rt_id))) + rt = ResourceType.get_by_id(rt_id) or abort( + 404, ErrFormat.resource_type_not_found.format("id={}".format(rt_id))) Resource.get_by(resource_type_id=rt_id) and abort(400, ErrFormat.resource_type_cannot_delete) @@ -165,8 +167,8 @@ def get_items(rg_id): @staticmethod def add(name, type_id, app_id, uid=None): - ResourceGroup.get_by(name=name, resource_type_id=type_id, app_id=app_id) and \ - abort(400, ErrFormat.resource_group_exists.format(name)) + ResourceGroup.get_by(name=name, resource_type_id=type_id, app_id=app_id) and abort( + 400, ErrFormat.resource_group_exists.format(name)) rg = ResourceGroup.create(name=name, resource_type_id=type_id, app_id=app_id, uid=uid) AuditCRUD.add_resource_log(app_id, AuditOperateType.create, @@ -175,8 +177,8 @@ def add(name, type_id, app_id, uid=None): @staticmethod def update(rg_id, items): - rg = ResourceGroup.get_by_id(rg_id) or \ - abort(404, ErrFormat.resource_group_not_found.format("id={}".format(rg_id))) + rg = ResourceGroup.get_by_id(rg_id) or abort( + 404, ErrFormat.resource_group_not_found.format("id={}".format(rg_id))) existed = ResourceGroupItems.get_by(group_id=rg_id, to_dict=False) existed_ids = [i.resource_id for i in existed] @@ -196,8 +198,8 @@ def update(rg_id, items): @staticmethod def delete(rg_id): - rg = ResourceGroup.get_by_id(rg_id) or \ - abort(404, ErrFormat.resource_group_not_found.format("id={}".format(rg_id))) + rg = ResourceGroup.get_by_id(rg_id) or abort( + 404, ErrFormat.resource_group_not_found.format("id={}".format(rg_id))) origin = rg.to_dict() rg.soft_delete() @@ -258,7 +260,8 @@ def search(cls, q, u, app_id, resource_type_id=None, page=1, page_size=None): numfound = query.count() res = [i.to_dict() for i in query.offset((page - 1) * page_size).limit(page_size)] for i in res: - i['user'] = UserCache.get(i['uid']).nickname if i['uid'] else '' + user = UserCache.get(i['uid']) if i['uid'] else '' + i['user'] = user and user.nickname return numfound, res @@ -266,14 +269,13 @@ def search(cls, q, u, app_id, resource_type_id=None, page=1, page_size=None): def add(cls, name, type_id, app_id, uid=None): type_id = cls._parse_resource_type_id(type_id, app_id) - Resource.get_by(name=name, resource_type_id=type_id, app_id=app_id) and \ - abort(400, ErrFormat.resource_exists.format(name)) + Resource.get_by(name=name, resource_type_id=type_id, app_id=app_id) and abort( + 400, ErrFormat.resource_exists.format(name)) r = Resource.create(name=name, resource_type_id=type_id, app_id=app_id, uid=uid) from api.tasks.acl import apply_trigger triggers = TriggerCRUD.match_triggers(app_id, r.name, r.resource_type_id, uid) - current_app.logger.info(triggers) for trigger in triggers: # auto trigger should be no uid apply_trigger.apply_async(args=(trigger.id,), diff --git a/cmdb-api/api/lib/perm/acl/resp_format.py b/cmdb-api/api/lib/perm/acl/resp_format.py index 114010d1..25f6bdcf 100644 --- a/cmdb-api/api/lib/perm/acl/resp_format.py +++ b/cmdb-api/api/lib/perm/acl/resp_format.py @@ -17,6 +17,7 @@ class ErrFormat(CommonErrFormat): role_exists = "角色 {} 已经存在!" global_role_not_found = "全局角色 {} 不存在!" global_role_exists = "全局角色 {} 已经存在!" + user_role_delete_invalid = "删除用户角色, 请在 用户管理 页面操作!" resource_no_permission = "您没有资源: {} 的 {} 权限" admin_required = "需要管理员权限" diff --git a/cmdb-api/api/lib/perm/acl/role.py b/cmdb-api/api/lib/perm/acl/role.py index 470e748a..0c2a28c9 100644 --- a/cmdb-api/api/lib/perm/acl/role.py +++ b/cmdb-api/api/lib/perm/acl/role.py @@ -6,6 +6,7 @@ import six from flask import abort from flask import current_app +from sqlalchemy import or_ from api.extensions import db from api.lib.perm.acl.app import AppCRUD @@ -212,18 +213,15 @@ class RoleCRUD(object): @staticmethod def search(q, app_id, page=1, page_size=None, user_role=True, is_all=False, user_only=False): - query = db.session.query(Role).filter(Role.deleted.is_(False)) - query1 = query.filter(Role.app_id == app_id).filter(Role.uid.is_(None)) - query2 = query.filter(Role.app_id.is_(None)).filter(Role.uid.is_(None)) - query = query1.union(query2) - - if user_role: - query1 = db.session.query(Role).filter(Role.deleted.is_(False)).filter(Role.uid.isnot(None)) - query = query.union(query1) - - if user_only: + if user_only: # only user role query = db.session.query(Role).filter(Role.deleted.is_(False)).filter(Role.uid.isnot(None)) + else: + query = db.session.query(Role).filter(Role.deleted.is_(False)).filter( + or_(Role.app_id == app_id, Role.app_id.is_(None))) + if not user_role: # only virtual role + query = query.filter(Role.uid.is_(None)) + if not is_all: role_ids = list(HasResourceRoleCache.get(app_id).keys()) query = query.filter(Role.id.in_(role_ids)) @@ -272,6 +270,13 @@ def update_role(rid, **kwargs): RoleCache.clean(rid) role = role.update(**kwargs) + + if origin['uid'] and kwargs.get('name') and kwargs.get('name') != origin['name']: + from api.models.acl import User + user = User.get_by(uid=origin['uid'], first=True, to_dict=False) + if user: + user.update(username=kwargs['name']) + AuditCRUD.add_role_log(role.app_id, AuditOperateType.update, AuditScope.role, role.id, origin, role.to_dict(), {}, ) @@ -286,14 +291,15 @@ def get_by_name(name, app_id): return role @classmethod - def delete_role(cls, rid): + def delete_role(cls, rid, force=False): from api.lib.perm.acl.acl import is_admin role = Role.get_by_id(rid) or abort(404, ErrFormat.role_not_found.format("rid={}".format(rid))) - if not role.app_id and not is_admin(): return abort(403, ErrFormat.admin_required) + not force and role.uid and abort(400, ErrFormat.user_role_delete_invalid) + origin = role.to_dict() child_ids = [] diff --git a/cmdb-api/api/lib/perm/acl/trigger.py b/cmdb-api/api/lib/perm/acl/trigger.py index f5a5d3f9..8035e4fe 100644 --- a/cmdb-api/api/lib/perm/acl/trigger.py +++ b/cmdb-api/api/lib/perm/acl/trigger.py @@ -6,9 +6,10 @@ import re from fnmatch import fnmatch -from flask import abort, current_app +from flask import abort -from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType +from api.lib.perm.acl.audit import AuditCRUD +from api.lib.perm.acl.audit import AuditOperateType from api.lib.perm.acl.cache import UserCache from api.lib.perm.acl.const import ACL_QUEUE from api.lib.perm.acl.resp_format import ErrFormat diff --git a/cmdb-api/api/lib/perm/acl/user.py b/cmdb-api/api/lib/perm/acl/user.py index 733cb0f6..d6f2fdd7 100644 --- a/cmdb-api/api/lib/perm/acl/user.py +++ b/cmdb-api/api/lib/perm/acl/user.py @@ -6,10 +6,12 @@ import uuid from flask import abort -from flask import g +from flask_login import current_user from api.extensions import db -from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditScope +from api.lib.perm.acl.audit import AuditCRUD +from api.lib.perm.acl.audit import AuditOperateType +from api.lib.perm.acl.audit import AuditScope from api.lib.perm.acl.cache import UserCache from api.lib.perm.acl.resp_format import ErrFormat from api.lib.perm.acl.role import RoleCRUD @@ -39,26 +41,31 @@ def gen_key_secret(): @classmethod def add(cls, **kwargs): - existed = User.get_by(username=kwargs['username'], email=kwargs['email']) + existed = User.get_by(username=kwargs['username']) existed and abort(400, ErrFormat.user_exists.format(kwargs['username'])) + existed = User.get_by(username=kwargs['email']) + existed and abort(400, ErrFormat.user_exists.format(kwargs['email'])) + kwargs['nickname'] = kwargs.get('nickname') or kwargs['username'] kwargs['block'] = 0 kwargs['key'], kwargs['secret'] = cls.gen_key_secret() - user_employee = db.session.query(User).filter(User.deleted.is_(False)).order_by( - User.employee_id.desc()).first() + user_employee = db.session.query(User).filter(User.deleted.is_(False)).order_by(User.employee_id.desc()).first() - biggest_employee_id = int(float(user_employee.employee_id)) \ - if user_employee is not None else 0 + biggest_employee_id = int(float(user_employee.employee_id)) if user_employee is not None else 0 kwargs['employee_id'] = '{0:04d}'.format(biggest_employee_id + 1) user = User.create(**kwargs) - RoleCRUD.add_role(user.username, uid=user.uid) + role = RoleCRUD.add_role(user.username, uid=user.uid) AuditCRUD.add_role_log(None, AuditOperateType.create, AuditScope.user, user.uid, {}, user.to_dict(), {}, {} ) + from api.lib.common_setting.employee import EmployeeCRUD + payload = {column: getattr(user, column) for column in ['uid', 'username', 'nickname', 'email', 'block']} + payload['rid'] = role.id + EmployeeCRUD.add_employee_from_acl_created(**payload) return user @@ -90,9 +97,9 @@ def update(uid, **kwargs): @classmethod def reset_key_secret(cls): key, secret = cls.gen_key_secret() - g.user.update(key=key, secret=secret) + current_user.update(key=key, secret=secret) - UserCache.clean(g.user) + UserCache.clean(current_user) return key, secret @@ -103,10 +110,14 @@ def delete(cls, uid): origin = user.to_dict() - user.soft_delete() + user.delete() UserCache.clean(user) + role = RoleCRUD.get_by_name(user.username, app_id=None) + if role: + RoleCRUD.delete_role(role[0]['id'], force=True) + AuditCRUD.add_role_log(None, AuditOperateType.delete, AuditScope.user, user.uid, origin, {}, {}, {}) diff --git a/cmdb-api/api/lib/perm/auth.py b/cmdb-api/api/lib/perm/auth.py index 7f53e773..76e24818 100644 --- a/cmdb-api/api/lib/perm/auth.py +++ b/cmdb-api/api/lib/perm/auth.py @@ -7,7 +7,6 @@ import jwt from flask import abort from flask import current_app -from flask import g from flask import request from flask import session from flask_login import login_user @@ -64,12 +63,10 @@ def _auth_with_key(): def _auth_with_session(): - if isinstance(getattr(g, 'user', None), User): - login_user(g.user) - return True if "acl" in session and "userName" in (session["acl"] or {}): login_user(UserCache.get(session["acl"]["userName"])) return True + return False @@ -108,7 +105,7 @@ def _auth_with_ip_white_list(): def _auth_with_app_token(): - if _auth_with_session(): + if _auth_with_session() or _auth_with_token(): if not is_app_admin(request.values.get('app_id')) and request.method != "GET": return False elif is_app_admin(request.values.get('app_id')): @@ -157,7 +154,7 @@ def _auth_with_acl_token(): def auth_required(func): - if request.json is not None: + if request.get_json(silent=True) is not None: setattr(request, 'values', request.json) else: setattr(request, 'values', request.values.to_dict()) diff --git a/cmdb-api/api/lib/resp_format.py b/cmdb-api/api/lib/resp_format.py index e3cf4da6..5a7c8525 100644 --- a/cmdb-api/api/lib/resp_format.py +++ b/cmdb-api/api/lib/resp_format.py @@ -9,6 +9,8 @@ class CommonErrFormat(object): not_found = "不存在" + circular_dependency_error = "存在循环依赖!" + unknown_search_error = "未知搜索错误" invalid_json = "json格式似乎不正确了, 请仔细确认一下!" diff --git a/cmdb-api/api/lib/secrets/__init__.py b/cmdb-api/api/lib/secrets/__init__.py new file mode 100644 index 00000000..380474e0 --- /dev/null +++ b/cmdb-api/api/lib/secrets/__init__.py @@ -0,0 +1 @@ +# -*- coding:utf-8 -*- diff --git a/cmdb-api/api/lib/secrets/inner.py b/cmdb-api/api/lib/secrets/inner.py new file mode 100644 index 00000000..2b577a68 --- /dev/null +++ b/cmdb-api/api/lib/secrets/inner.py @@ -0,0 +1,430 @@ +import os +import secrets +import sys +from base64 import b64decode, b64encode + +from Cryptodome.Protocol.SecretSharing import Shamir +from colorama import Back +from colorama import Fore +from colorama import Style +from colorama import init as colorama_init +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes +from cryptography.hazmat.primitives import padding +from cryptography.hazmat.primitives.ciphers import Cipher +from cryptography.hazmat.primitives.ciphers import algorithms +from cryptography.hazmat.primitives.ciphers import modes +from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC +from flask import current_app + +global_iv_length = 16 +global_key_shares = 5 # Number of generated key shares +global_key_threshold = 3 # Minimum number of shares required to rebuild the key + +backend_root_key_name = "root_key" +backend_encrypt_key_name = "encrypt_key" +backend_root_key_salt_name = "root_key_salt" +backend_encrypt_key_salt_name = "encrypt_key_salt" +backend_seal_key = "seal_status" +success = "success" +seal_status = True + + +def string_to_bytes(value): + if isinstance(value, bytes): + return value + if sys.version_info.major == 2: + byte_string = value + else: + byte_string = value.encode("utf-8") + + return byte_string + + +class Backend: + def __init__(self, backend=None): + self.backend = backend + + def get(self, key): + return self.backend.get(key) + + def add(self, key, value): + return self.backend.add(key, value) + + def update(self, key, value): + return self.backend.update(key, value) + + +class KeyManage: + + def __init__(self, trigger=None, backend=None): + self.trigger = trigger + self.backend = backend + if backend: + self.backend = Backend(backend) + + def init_app(self, app, backend=None): + if (sys.argv[0].endswith("gunicorn") or + (len(sys.argv) > 1 and sys.argv[1] in ("run", "cmdb-password-data-migrate"))): + self.trigger = app.config.get("INNER_TRIGGER_TOKEN") + if not self.trigger: + return + + self.backend = backend + resp = self.auto_unseal() + self.print_response(resp) + + def hash_root_key(self, value): + algorithm = hashes.SHA256() + salt = self.backend.get(backend_root_key_salt_name) + if not salt: + salt = secrets.token_hex(16) + msg, ok = self.backend.add(backend_root_key_salt_name, salt) + if not ok: + return msg, ok + + kdf = PBKDF2HMAC( + algorithm=algorithm, + length=32, + salt=string_to_bytes(salt), + iterations=100000, + ) + key = kdf.derive(string_to_bytes(value)) + + return b64encode(key).decode('utf-8'), True + + def generate_encrypt_key(self, key): + algorithm = hashes.SHA256() + salt = self.backend.get(backend_encrypt_key_salt_name) + if not salt: + salt = secrets.token_hex(32) + + kdf = PBKDF2HMAC( + algorithm=algorithm, + length=32, + salt=string_to_bytes(salt), + iterations=100000, + backend=default_backend() + ) + key = kdf.derive(string_to_bytes(key)) + msg, ok = self.backend.add(backend_encrypt_key_salt_name, salt) + if ok: + return b64encode(key).decode('utf-8'), ok + else: + return msg, ok + + @classmethod + def generate_keys(cls, secret): + shares = Shamir.split(global_key_threshold, global_key_shares, secret, False) + new_shares = [] + for share in shares: + t = [i for i in share[1]] + [ord(i) for i in "{:0>2}".format(share[0])] + new_shares.append(b64encode(bytes(t))) + + return new_shares + + def is_valid_root_key(self, root_key): + root_key_hash, ok = self.hash_root_key(root_key) + if not ok: + return root_key_hash, ok + backend_root_key_hash = self.backend.get(backend_root_key_name) + if not backend_root_key_hash: + return "should init firstly", False + elif backend_root_key_hash != root_key_hash: + return "invalid root key", False + else: + return "", True + + def auth_root_secret(self, root_key): + msg, ok = self.is_valid_root_key(root_key) + if not ok: + return { + "message": msg, + "status": "failed" + } + + encrypt_key_aes = self.backend.get(backend_encrypt_key_name) + if not encrypt_key_aes: + return { + "message": "encrypt key is empty", + "status": "failed" + } + + secrets_encrypt_key, ok = InnerCrypt.aes_decrypt(string_to_bytes(root_key), encrypt_key_aes) + if ok: + msg, ok = self.backend.update(backend_seal_key, "open") + if ok: + current_app.config["secrets_encrypt_key"] = secrets_encrypt_key + current_app.config["secrets_root_key"] = root_key + current_app.config["secrets_shares"] = [] + return {"message": success, "status": success} + return {"message": msg, "status": "failed"} + else: + return { + "message": secrets_encrypt_key, + "status": "failed" + } + + def unseal(self, key): + if not self.is_seal(): + return { + "message": "current status is unseal, skip", + "status": "skip" + } + + try: + t = [i for i in b64decode(key)] + v = (int("".join([chr(i) for i in t[-2:]])), bytes(t[:-2])) + shares = current_app.config.get("secrets_shares", []) + if v not in shares: + shares.append(v) + current_app.config["secrets_shares"] = shares + + if len(shares) >= global_key_threshold: + recovered_secret = Shamir.combine(shares[:global_key_threshold], False) + return self.auth_root_secret(b64encode(recovered_secret)) + else: + return { + "message": "waiting for inputting other unseal key {0}/{1}".format(len(shares), + global_key_threshold), + "status": "waiting" + } + except Exception as e: + return { + "message": "invalid token: " + str(e), + "status": "failed" + } + + def generate_unseal_keys(self): + info = self.backend.get(backend_root_key_name) + if info: + return "already exist", [], False + + secret = AESGCM.generate_key(128) + shares = self.generate_keys(secret) + + return b64encode(secret), shares, True + + def init(self): + """ + init the master key, unseal key and store in backend + :return: + """ + root_key = self.backend.get(backend_root_key_name) + if root_key: + return {"message": "already init, skip", "status": "skip"}, False + else: + root_key, shares, status = self.generate_unseal_keys() + if not status: + return {"message": root_key, "status": "failed"}, False + + # hash root key and store in backend + root_key_hash, ok = self.hash_root_key(root_key) + if not ok: + return {"message": root_key_hash, "status": "failed"}, False + + msg, ok = self.backend.add(backend_root_key_name, root_key_hash) + if not ok: + return {"message": msg, "status": "failed"}, False + + # generate encrypt key from root_key and store in backend + encrypt_key, ok = self.generate_encrypt_key(root_key) + if not ok: + return {"message": encrypt_key, "status": "failed"} + + encrypt_key_aes, status = InnerCrypt.aes_encrypt(root_key, encrypt_key) + if not status: + return {"message": encrypt_key_aes, "status": "failed"} + + msg, ok = self.backend.add(backend_encrypt_key_name, encrypt_key_aes) + if not ok: + return {"message": msg, "status": "failed"}, False + msg, ok = self.backend.add(backend_seal_key, "open") + if not ok: + return {"message": msg, "status": "failed"}, False + current_app.config["secrets_root_key"] = root_key + current_app.config["secrets_encrypt_key"] = encrypt_key + self.print_token(shares, root_token=root_key) + + return {"message": "OK", + "details": { + "root_token": root_key, + "seal_tokens": shares, + }}, True + + def auto_unseal(self): + if not self.trigger: + return { + "message": "trigger config is empty, skip", + "status": "skip" + } + + if self.trigger.startswith("http"): + return { + "message": "todo in next step, skip", + "status": "skip" + } + # TODO + elif len(self.trigger.strip()) == 24: + res = self.auth_root_secret(self.trigger.encode()) + if res.get("status") == success: + return { + "message": success, + "status": success + } + else: + return { + "message": res.get("message"), + "status": "failed" + } + else: + return { + "message": "trigger config is invalid, skip", + "status": "skip" + } + + def seal(self, root_key): + root_key = root_key.encode() + msg, ok = self.is_valid_root_key(root_key) + if not ok: + return { + "message": msg, + "status": "failed" + } + else: + msg, ok = self.backend.update(backend_seal_key, "block") + if not ok: + return { + "message": msg, + "status": "failed", + } + current_app.config["secrets_root_key"] = '' + current_app.config["secrets_encrypt_key"] = '' + return { + "message": success, + "status": success + } + + def is_seal(self): + """ + If there is no initialization or the root key is inconsistent, it is considered to be in a sealed state. + :return: + """ + secrets_root_key = current_app.config.get("secrets_root_key") + msg, ok = self.is_valid_root_key(secrets_root_key) + if not ok: + return {"message": msg, "status": "failed"} + status = self.backend.get(backend_seal_key) + return status == "block" + + @classmethod + def print_token(cls, shares, root_token): + """ + data: {"message": "OK", + "details": { + "root_token": root_key, + "seal_tokens": shares, + }} + """ + colorama_init() + print(Style.BRIGHT, "Please be sure to store the Unseal Key in a secure location and avoid losing it." + " The Unseal Key is required to unseal the system every time when it restarts." + " Successful unsealing is necessary to enable the password feature." + Style.RESET_ALL) + + for i, v in enumerate(shares): + print( + "unseal token " + str(i + 1) + ": " + Fore.RED + Back.BLACK + v.decode("utf-8") + Style.RESET_ALL) + print() + + print(Fore.GREEN + "root token: " + root_token.decode("utf-8") + Style.RESET_ALL) + + @classmethod + def print_response(cls, data): + status = data.get("status", "") + message = data.get("message", "") + status_colors = { + "skip": Style.BRIGHT, + "failed": Fore.RED, + "waiting": Fore.YELLOW, + } + print(status_colors.get(status, Fore.GREEN), message, Style.RESET_ALL) + + +class InnerCrypt: + def __init__(self): + secrets_encrypt_key = current_app.config.get("secrets_encrypt_key", "") + self.encrypt_key = b64decode(secrets_encrypt_key.encode("utf-8")) + + def encrypt(self, plaintext): + """ + encrypt method contain aes currently + """ + return self.aes_encrypt(self.encrypt_key, plaintext) + + def decrypt(self, ciphertext): + """ + decrypt method contain aes currently + """ + return self.aes_decrypt(self.encrypt_key, ciphertext) + + @classmethod + def aes_encrypt(cls, key, plaintext): + if isinstance(plaintext, str): + plaintext = string_to_bytes(plaintext) + iv = os.urandom(global_iv_length) + try: + cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend()) + encryptor = cipher.encryptor() + v_padder = padding.PKCS7(algorithms.AES.block_size).padder() + padded_plaintext = v_padder.update(plaintext) + v_padder.finalize() + ciphertext = encryptor.update(padded_plaintext) + encryptor.finalize() + + return b64encode(iv + ciphertext).decode("utf-8"), True + except Exception as e: + return str(e), False + + @classmethod + def aes_decrypt(cls, key, ciphertext): + try: + s = b64decode(ciphertext.encode("utf-8")) + iv = s[:global_iv_length] + ciphertext = s[global_iv_length:] + cipher = Cipher(algorithms.AES(key), modes.CBC(iv), backend=default_backend()) + decrypter = cipher.decryptor() + decrypted_padded_plaintext = decrypter.update(ciphertext) + decrypter.finalize() + unpadder = padding.PKCS7(algorithms.AES.block_size).unpadder() + plaintext = unpadder.update(decrypted_padded_plaintext) + unpadder.finalize() + + return plaintext.decode('utf-8'), True + except Exception as e: + return str(e), False + + +if __name__ == "__main__": + + km = KeyManage() + # info, shares, status = km.generate_unseal_keys() + # print(info, shares, status) + # print("..................") + # for i in shares: + # print(b64encode(i[1]).decode()) + + res1, ok1 = km.init() + if not ok1: + print(res1) + # for j in res["details"]["seal_tokens"]: + # r = km.unseal(j) + # if r["status"] != "waiting": + # if r["status"] != "success": + # print("r........", r) + # else: + # print(r) + # break + + t_plaintext = b"Hello, World!" # The plaintext to encrypt + c = InnerCrypt() + t_ciphertext, status1 = c.encrypt(t_plaintext) + print("Ciphertext:", t_ciphertext) + decrypted_plaintext, status2 = c.decrypt(t_ciphertext) + print("Decrypted plaintext:", decrypted_plaintext) diff --git a/cmdb-api/api/lib/secrets/secrets.py b/cmdb-api/api/lib/secrets/secrets.py new file mode 100644 index 00000000..674f570c --- /dev/null +++ b/cmdb-api/api/lib/secrets/secrets.py @@ -0,0 +1,35 @@ +from api.models.cmdb import InnerKV + + +class InnerKVManger(object): + def __init__(self): + pass + + @classmethod + def add(cls, key, value): + data = {"key": key, "value": value} + res = InnerKV.create(**data) + if res.key == key: + return "success", True + + return "add failed", False + + @classmethod + def get(cls, key): + res = InnerKV.get_by(first=True, to_dict=False, key=key) + if not res: + return None + + return res.value + + @classmethod + def update(cls, key, value): + res = InnerKV.get_by(first=True, to_dict=False, key=key) + if not res: + return cls.add(key, value) + + t = res.update(value=value) + if t.key == key: + return "success", True + + return "update failed", True diff --git a/cmdb-api/api/lib/secrets/vault.py b/cmdb-api/api/lib/secrets/vault.py new file mode 100644 index 00000000..a5746f55 --- /dev/null +++ b/cmdb-api/api/lib/secrets/vault.py @@ -0,0 +1,141 @@ +from base64 import b64decode +from base64 import b64encode + +import hvac + + +class VaultClient: + def __init__(self, base_url, token, mount_path='cmdb'): + self.client = hvac.Client(url=base_url, token=token) + self.mount_path = mount_path + + def create_app_role(self, role_name, policies): + resp = self.client.create_approle(role_name, policies=policies) + + return resp == 200 + + def delete_app_role(self, role_name): + resp = self.client.delete_approle(role_name) + + return resp == 204 + + def update_app_role_policies(self, role_name, policies): + resp = self.client.update_approle_role(role_name, policies=policies) + + return resp == 204 + + def get_app_role(self, role_name): + resp = self.client.get_approle(role_name) + resp.json() + if resp.status_code == 200: + return resp.json + else: + return {} + + def enable_secrets_engine(self): + resp = self.client.sys.enable_secrets_engine('kv', path=self.mount_path) + resp_01 = self.client.sys.enable_secrets_engine('transit') + + if resp.status_code == 200 and resp_01.status_code == 200: + return resp.json + else: + return {} + + def encrypt(self, plaintext): + response = self.client.secrets.transit.encrypt_data(name='transit-key', plaintext=plaintext) + ciphertext = response['data']['ciphertext'] + + return ciphertext + + # decrypt data + def decrypt(self, ciphertext): + response = self.client.secrets.transit.decrypt_data(name='transit-key', ciphertext=ciphertext) + plaintext = response['data']['plaintext'] + + return plaintext + + def write(self, path, data, encrypt=None): + if encrypt: + for k, v in data.items(): + data[k] = self.encrypt(self.encode_base64(v)) + response = self.client.secrets.kv.v2.create_or_update_secret( + path=path, + secret=data, + mount_point=self.mount_path + ) + + return response + + # read data + def read(self, path, decrypt=True): + try: + response = self.client.secrets.kv.v2.read_secret_version( + path=path, raise_on_deleted_version=False, mount_point=self.mount_path + ) + except Exception as e: + return str(e), False + data = response['data']['data'] + if decrypt: + try: + for k, v in data.items(): + data[k] = self.decode_base64(self.decrypt(v)) + except: + return data, True + + return data, True + + # update data + def update(self, path, data, overwrite=True, encrypt=True): + if encrypt: + for k, v in data.items(): + data[k] = self.encrypt(self.encode_base64(v)) + if overwrite: + response = self.client.secrets.kv.v2.create_or_update_secret( + path=path, + secret=data, + mount_point=self.mount_path + ) + else: + response = self.client.secrets.kv.v2.patch(path=path, secret=data, mount_point=self.mount_path) + + return response + + # delete data + def delete(self, path): + response = self.client.secrets.kv.v2.delete_metadata_and_all_versions( + path=path, + mount_point=self.mount_path + ) + + return response + + # Base64 encode + @classmethod + def encode_base64(cls, data): + encoded_bytes = b64encode(data.encode()) + encoded_string = encoded_bytes.decode() + + return encoded_string + + # Base64 decode + @classmethod + def decode_base64(cls, encoded_string): + decoded_bytes = b64decode(encoded_string) + decoded_string = decoded_bytes.decode() + + return decoded_string + + +if __name__ == "__main__": + _base_url = "http://localhost:8200" + _token = "your token" + + _path = "test001" + # Example + sdk = VaultClient(_base_url, _token) + # sdk.enable_secrets_engine() + _data = {"key1": "value1", "key2": "value2", "key3": "value3"} + _data = sdk.update(_path, _data, overwrite=True, encrypt=True) + print(_data) + _data = sdk.read(_path, decrypt=True) + print(_data) diff --git a/cmdb-api/api/lib/utils.py b/cmdb-api/api/lib/utils.py index b4dd7f84..eddc7b8f 100644 --- a/cmdb-api/api/lib/utils.py +++ b/cmdb-api/api/lib/utils.py @@ -1,7 +1,6 @@ # -*- coding:utf-8 -*- import base64 -import json import sys import time from typing import Set @@ -13,6 +12,9 @@ from elasticsearch import Elasticsearch from flask import current_app +from api.lib.secrets.inner import InnerCrypt +from api.lib.secrets.inner import KeyManage + class BaseEnum(object): _ALL_ = set() # type: Set[str] @@ -113,7 +115,7 @@ def delete(self, key_id, prefix): try: ret = self.r.hdel(prefix, key_id) if not ret: - current_app.logger.warn("[{0}] is not in redis".format(key_id)) + current_app.logger.warning("[{0}] is not in redis".format(key_id)) except Exception as e: current_app.logger.error("delete redis key error, {0}".format(str(e))) @@ -204,9 +206,9 @@ def read(self, query, filter_path=None): res = self.es.search(index=self.index, body=query, filter_path=filter_path) if res['hits'].get('hits'): - return res['hits']['total']['value'], \ - [i['_source'] for i in res['hits']['hits']], \ - res.get("aggregations", {}) + return (res['hits']['total']['value'], + [i['_source'] for i in res['hits']['hits']], + res.get("aggregations", {})) else: return 0, [], {} @@ -257,93 +259,10 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.release() -class Redis2Handler(object): - def __init__(self, flask_app=None, prefix=None): - self.flask_app = flask_app - self.prefix = prefix - self.r = None - - def init_app(self, app): - self.flask_app = app - config = self.flask_app.config - try: - pool = redis.ConnectionPool( - max_connections=config.get("REDIS_MAX_CONN"), - host=config.get("ONEAGENT_REDIS_HOST"), - port=config.get("ONEAGENT_REDIS_PORT"), - db=config.get("ONEAGENT_REDIS_DB"), - password=config.get("ONEAGENT_REDIS_PASSWORD") - ) - self.r = redis.Redis(connection_pool=pool) - except Exception as e: - current_app.logger.warning(str(e)) - current_app.logger.error("init redis connection failed") - - def get(self, key): - try: - value = json.loads(self.r.get(key)) - except: - return - - return value - - def lrange(self, key, start=0, end=-1): - try: - value = "".join(map(redis_decode, self.r.lrange(key, start, end) or [])) - except: - return - - return value - - def lrange2(self, key, start=0, end=-1): - try: - return list(map(redis_decode, self.r.lrange(key, start, end) or [])) - except: - return [] - - def llen(self, key): - try: - return self.r.llen(key) or 0 - except: - return 0 - - def hget(self, key, field): - try: - return self.r.hget(key, field) - except Exception as e: - current_app.logger.warning("hget redis failed, %s" % str(e)) - return - - def hset(self, key, field, value): - try: - self.r.hset(key, field, value) - except Exception as e: - current_app.logger.warning("hset redis failed, %s" % str(e)) - return - - def expire(self, key, timeout): - try: - self.r.expire(key, timeout) - except Exception as e: - current_app.logger.warning("expire redis failed, %s" % str(e)) - return - - -def redis_decode(x): - try: - return x.decode() - except Exception as e: - print(x, e) - try: - return x.decode("gb18030") - except: - return "decode failed" - - class AESCrypto(object): BLOCK_SIZE = 16 # Bytes - pad = lambda s: s + (AESCrypto.BLOCK_SIZE - len(s) % AESCrypto.BLOCK_SIZE) * \ - chr(AESCrypto.BLOCK_SIZE - len(s) % AESCrypto.BLOCK_SIZE) + pad = lambda s: s + ((AESCrypto.BLOCK_SIZE - len(s) % AESCrypto.BLOCK_SIZE) * + chr(AESCrypto.BLOCK_SIZE - len(s) % AESCrypto.BLOCK_SIZE)) unpad = lambda s: s[:-ord(s[len(s) - 1:])] iv = '0102030405060708' @@ -352,7 +271,7 @@ class AESCrypto(object): def key(): key = current_app.config.get("SECRET_KEY")[:16] if len(key) < 16: - key = "{}{}".format(key, (16 - len(key) * "x")) + key = "{}{}".format(key, (16 - len(key)) * "x") return key.encode('utf8') @@ -370,3 +289,33 @@ def decrypt(cls, data): text_decrypted = cipher.decrypt(encode_bytes) return cls.unpad(text_decrypted).decode('utf8') + + +class Crypto(AESCrypto): + @classmethod + def encrypt(cls, data): + from api.lib.secrets.secrets import InnerKVManger + + if not KeyManage(backend=InnerKVManger()).is_seal(): + res, status = InnerCrypt().encrypt(data) + if status: + return res + + return AESCrypto().encrypt(data) + + @classmethod + def decrypt(cls, data): + from api.lib.secrets.secrets import InnerKVManger + + if not KeyManage(backend=InnerKVManger()).is_seal(): + try: + res, status = InnerCrypt().decrypt(data) + if status: + return res + except: + pass + + try: + return AESCrypto().decrypt(data) + except: + return data diff --git a/cmdb-api/api/lib/webhook.py b/cmdb-api/api/lib/webhook.py new file mode 100644 index 00000000..a5133e09 --- /dev/null +++ b/cmdb-api/api/lib/webhook.py @@ -0,0 +1,109 @@ +# -*- coding:utf-8 -*- + +import json +from functools import partial + +import requests +from jinja2 import Template +from requests.auth import HTTPBasicAuth +from requests_oauthlib import OAuth2Session + + +class BearerAuth(requests.auth.AuthBase): + def __init__(self, token): + self.token = token + + def __call__(self, r): + r.headers["authorization"] = "Bearer {}".format(self.token) + return r + + +def _wrap_auth(**kwargs): + auth_type = (kwargs.get('type') or "").lower() + if auth_type == "basicauth": + return HTTPBasicAuth(kwargs.get('username'), kwargs.get('password')) + + elif auth_type == "bearer": + return BearerAuth(kwargs.get('token')) + + elif auth_type == 'oauth2.0': + client_id = kwargs.get('client_id') + client_secret = kwargs.get('client_secret') + authorization_base_url = kwargs.get('authorization_base_url') + token_url = kwargs.get('token_url') + redirect_url = kwargs.get('redirect_url') + scope = kwargs.get('scope') + + oauth2_session = OAuth2Session(client_id, scope=scope or None) + oauth2_session.authorization_url(authorization_base_url) + + oauth2_session.fetch_token(token_url, client_secret=client_secret, authorization_response=redirect_url) + + return oauth2_session + + elif auth_type == "apikey": + return HTTPBasicAuth(kwargs.get('key'), kwargs.get('value')) + + +def webhook_request(webhook, payload): + """ + + :param webhook: + { + "url": "https://veops.cn" + "method": "GET|POST|PUT|DELETE" + "body": {}, + "headers": { + "Content-Type": "Application/json" + }, + "parameters": { + "key": "value" + }, + "authorization": { + "type": "BasicAuth|Bearer|OAuth2.0|APIKey", + "password": "mmmm", # BasicAuth + "username": "bbb", # BasicAuth + + "token": "xxx", # Bearer + + "key": "xxx", # APIKey + "value": "xxx", # APIKey + + "client_id": "xxx", # OAuth2.0 + "client_secret": "xxx", # OAuth2.0 + "authorization_base_url": "xxx", # OAuth2.0 + "token_url": "xxx", # OAuth2.0 + "redirect_url": "xxx", # OAuth2.0 + "scope": "xxx" # OAuth2.0 + } + } + :param payload: + :return: + """ + assert webhook.get('url') is not None + + payload = {k: v or '' for k, v in payload.items()} + + url = Template(webhook['url']).render(payload) + + params = webhook.get('parameters') or None + if isinstance(params, dict): + params = json.loads(Template(json.dumps(params)).render(payload)) + + headers = json.loads(Template(json.dumps(webhook.get('headers') or {})).render(payload)) + + data = Template(json.dumps(webhook.get('body', ''))).render(payload) + auth = _wrap_auth(**webhook.get('authorization', {})) + + if (webhook.get('authorization', {}).get("type") or '').lower() == 'oauth2.0': + request = getattr(auth, webhook.get('method', 'GET').lower()) + else: + request = partial(requests.request, webhook.get('method', 'GET')) + + return request( + url, + params=params, + headers=headers or None, + data=data, + auth=auth + ) diff --git a/cmdb-api/api/models/acl.py b/cmdb-api/api/models/acl.py index ebf02fff..b0683ece 100644 --- a/cmdb-api/api/models/acl.py +++ b/cmdb-api/api/models/acl.py @@ -5,7 +5,8 @@ import hashlib from datetime import datetime -import ldap +from ldap3 import Server, Connection, ALL +from ldap3.core.exceptions import LDAPBindError, LDAPCertificateError from flask import current_app from flask_sqlalchemy import BaseQuery @@ -57,24 +58,25 @@ def authenticate_with_key(self, key, secret, args, path): return user, authenticated def authenticate_with_ldap(self, username, password): - ldap_conn = ldap.initialize(current_app.config.get('LDAP_SERVER')) - ldap_conn.protocol_version = 3 - ldap_conn.set_option(ldap.OPT_REFERRALS, 0) + server = Server(current_app.config.get('LDAP_SERVER'), get_info=ALL) if '@' in username: email = username - who = '{0}@{1}'.format(username.split('@')[0], current_app.config.get('LDAP_DOMAIN')) + who = current_app.config.get('LDAP_USER_DN').format(username.split('@')[0]) else: - who = '{0}@{1}'.format(username, current_app.config.get('LDAP_DOMAIN')) - email = who + who = current_app.config.get('LDAP_USER_DN').format(username) + email = "{}@{}".format(who, current_app.config.get('LDAP_DOMAIN')) username = username.split('@')[0] user = self.get_by_username(username) try: - if not password: - raise ldap.INVALID_CREDENTIALS + raise LDAPCertificateError - ldap_conn.simple_bind_s(who, password) + conn = Connection(server, user=who, password=password) + conn.bind() + if conn.result['result'] != 0: + raise LDAPBindError + conn.unbind() if not user: from api.lib.perm.acl.user import UserCRUD @@ -84,7 +86,7 @@ def authenticate_with_ldap(self, username, password): op_record.apply_async(args=(None, username, OperateType.LOGIN, ["ACL"]), queue=ACL_QUEUE) return user, True - except ldap.INVALID_CREDENTIALS: + except LDAPBindError: return user, False def search(self, key): diff --git a/cmdb-api/api/models/cmdb.py b/cmdb-api/api/models/cmdb.py index e1693cb8..a17a1d3c 100644 --- a/cmdb-api/api/models/cmdb.py +++ b/cmdb-api/api/models/cmdb.py @@ -12,7 +12,9 @@ from api.lib.cmdb.const import ConstraintEnum from api.lib.cmdb.const import OperateType from api.lib.cmdb.const import ValueTypeEnum -from api.lib.database import Model, Model2 +from api.lib.database import Model +from api.lib.database import Model2 +from api.lib.utils import Crypto # template @@ -89,12 +91,37 @@ class Attribute(Model): compute_expr = db.Column(db.Text) compute_script = db.Column(db.Text) - choice_web_hook = db.Column(db.JSON) + _choice_web_hook = db.Column('choice_web_hook', db.JSON) + choice_other = db.Column(db.JSON) uid = db.Column(db.Integer, index=True) option = db.Column(db.JSON) + def _get_webhook(self): + if self._choice_web_hook: + if self._choice_web_hook.get('headers') and "Cookie" in self._choice_web_hook['headers']: + self._choice_web_hook['headers']['Cookie'] = Crypto.decrypt(self._choice_web_hook['headers']['Cookie']) + + if self._choice_web_hook.get('authorization'): + for k, v in self._choice_web_hook['authorization'].items(): + self._choice_web_hook['authorization'][k] = Crypto.decrypt(v) + + return self._choice_web_hook + + def _set_webhook(self, data): + if data: + if data.get('headers') and "Cookie" in data['headers']: + data['headers']['Cookie'] = Crypto.encrypt(data['headers']['Cookie']) + + if data.get('authorization'): + for k, v in data['authorization'].items(): + data['authorization'][k] = Crypto.encrypt(v) + + self._choice_web_hook = data + + choice_web_hook = db.synonym("_choice_web_hook", descriptor=property(_get_webhook, _set_webhook)) + class CITypeAttribute(Model): __tablename__ = "c_ci_type_attributes" @@ -125,16 +152,45 @@ class CITypeAttributeGroupItem(Model): class CITypeTrigger(Model): - # __tablename__ = "c_ci_type_triggers" __tablename__ = "c_c_t_t" type_id = db.Column(db.Integer, db.ForeignKey('c_ci_types.id'), nullable=False) - attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id"), nullable=False) - notify = db.Column(db.JSON) # {subject: x, body: x, wx_to: [], mail_to: [], before_days: 0, notify_at: 08:00} + attr_id = db.Column(db.Integer, db.ForeignKey("c_attributes.id")) + _option = db.Column('notify', db.JSON) + + def _get_option(self): + if self._option and self._option.get('webhooks'): + if self._option['webhooks'].get('authorization'): + for k, v in self._option['webhooks']['authorization'].items(): + self._option['webhooks']['authorization'][k] = Crypto.decrypt(v) + + return self._option + + def _set_option(self, data): + if data and data.get('webhooks'): + if data['webhooks'].get('authorization'): + for k, v in data['webhooks']['authorization'].items(): + data['webhooks']['authorization'][k] = Crypto.encrypt(v) + + self._option = data + + option = db.synonym("_option", descriptor=property(_get_option, _set_option)) + + +class CITriggerHistory(Model): + __tablename__ = "c_ci_trigger_histories" + + operate_type = db.Column(db.Enum(*OperateType.all(), name="operate_type")) + record_id = db.Column(db.Integer, db.ForeignKey("c_records.id")) + ci_id = db.Column(db.Integer, index=True, nullable=False) + trigger_id = db.Column(db.Integer, db.ForeignKey("c_c_t_t.id")) + trigger_name = db.Column(db.String(64)) + is_ok = db.Column(db.Boolean, default=False) + notify = db.Column(db.Text) + webhook = db.Column(db.Text) class CITypeUniqueConstraint(Model): - # __tablename__ = "c_ci_type_unique_constraints" __tablename__ = "c_c_t_u_c" type_id = db.Column(db.Integer, db.ForeignKey('c_ci_types.id'), nullable=False) @@ -250,6 +306,9 @@ class CIIndexValueDateTime(Model): class CIValueInteger(Model): + """ + Deprecated in a future version + """ __tablename__ = "c_value_integers" ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) @@ -261,6 +320,9 @@ class CIValueInteger(Model): class CIValueFloat(Model): + """ + Deprecated in a future version + """ __tablename__ = "c_value_floats" ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) @@ -283,6 +345,9 @@ class CIValueText(Model): class CIValueDateTime(Model): + """ + Deprecated in a future version + """ __tablename__ = "c_value_datetime" ci_id = db.Column(db.Integer, db.ForeignKey('c_cis.id'), nullable=False) @@ -354,7 +419,6 @@ class CITypeHistory(Model): # preference class PreferenceShowAttributes(Model): - # __tablename__ = "c_preference_show_attributes" __tablename__ = "c_psa" uid = db.Column(db.Integer, index=True, nullable=False) @@ -368,7 +432,6 @@ class PreferenceShowAttributes(Model): class PreferenceTreeView(Model): - # __tablename__ = "c_preference_tree_views" __tablename__ = "c_ptv" uid = db.Column(db.Integer, index=True, nullable=False) @@ -377,7 +440,6 @@ class PreferenceTreeView(Model): class PreferenceRelationView(Model): - # __tablename__ = "c_preference_relation_views" __tablename__ = "c_prv" uid = db.Column(db.Integer, index=True, nullable=False) @@ -486,3 +548,10 @@ class CIFilterPerms(Model): attr_filter = db.Column(db.Text) rid = db.Column(db.Integer, index=True) + + +class InnerKV(Model): + __tablename__ = "c_kv" + + key = db.Column(db.String(128), index=True) + value = db.Column(db.Text) diff --git a/cmdb-api/api/models/common_setting.py b/cmdb-api/api/models/common_setting.py index 741f6644..a141ee8d 100644 --- a/cmdb-api/api/models/common_setting.py +++ b/cmdb-api/api/models/common_setting.py @@ -13,40 +13,41 @@ class Department(ModelWithoutPK): __tablename__ = 'common_department' department_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - department_name = db.Column(db.VARCHAR(255), default='', comment='部门名称') + department_name = db.Column(db.VARCHAR(255), default='') department_director_id = db.Column( - db.Integer, default=0, comment='部门负责人ID') - department_parent_id = db.Column(db.Integer, default=1, comment='上级部门ID') + db.Integer, default=0) + department_parent_id = db.Column(db.Integer, default=1) - sort_value = db.Column(db.Integer, default=0, comment='排序值') + sort_value = db.Column(db.Integer, default=0) - acl_rid = db.Column(db.Integer, comment='ACL中rid', default=0) + acl_rid = db.Column(db.Integer, default=0) class Employee(ModelWithoutPK): __tablename__ = 'common_employee' employee_id = db.Column(db.Integer, primary_key=True, autoincrement=True) - email = db.Column(db.VARCHAR(255), default='', comment='邮箱') - username = db.Column(db.VARCHAR(255), default='', comment='用户名') - nickname = db.Column(db.VARCHAR(255), default='', comment='姓名') - sex = db.Column(db.VARCHAR(64), default='', comment='性别') - position_name = db.Column(db.VARCHAR(255), default='', comment='职位名称') - mobile = db.Column(db.VARCHAR(255), default='', comment='电话号码') - avatar = db.Column(db.VARCHAR(255), default='', comment='头像') + email = db.Column(db.VARCHAR(255), default='') + username = db.Column(db.VARCHAR(255), default='') + nickname = db.Column(db.VARCHAR(255), default='') + sex = db.Column(db.VARCHAR(64), default='') + position_name = db.Column(db.VARCHAR(255), default='') + mobile = db.Column(db.VARCHAR(255), default='') + avatar = db.Column(db.VARCHAR(255), default='') - direct_supervisor_id = db.Column(db.Integer, default=0, comment='直接上级ID') + direct_supervisor_id = db.Column(db.Integer, default=0) department_id = db.Column(db.Integer, - db.ForeignKey('common_department.department_id'), - comment='部门ID', + db.ForeignKey('common_department.department_id') ) - acl_uid = db.Column(db.Integer, comment='ACL中uid', default=0) - acl_rid = db.Column(db.Integer, comment='ACL中rid', default=0) - acl_virtual_rid = db.Column(db.Integer, comment='ACL中虚拟角色rid', default=0) - last_login = db.Column(db.TIMESTAMP, nullable=True, comment='上次登录时间') - block = db.Column(db.Integer, comment='锁定状态', default=0) + acl_uid = db.Column(db.Integer, default=0) + acl_rid = db.Column(db.Integer, default=0) + acl_virtual_rid = db.Column(db.Integer, default=0) + last_login = db.Column(db.TIMESTAMP, nullable=True) + block = db.Column(db.Integer, default=0) + + notice_info = db.Column(db.JSON, default={}) _department = db.relationship( 'Department', backref='common_employee.department_id', @@ -55,14 +56,11 @@ class Employee(ModelWithoutPK): class EmployeeInfo(Model): - """ - 员工信息 - """ __tablename__ = 'common_employee_info' - info = db.Column(db.JSON, default={}, comment='员工信息') + info = db.Column(db.JSON, default={}) employee_id = db.Column(db.Integer, db.ForeignKey( - 'common_employee.employee_id'), comment='员工ID') + 'common_employee.employee_id')) employee = db.relationship( 'Employee', backref='common_employee.employee_id', lazy='joined') @@ -74,16 +72,27 @@ class CompanyInfo(Model): class InternalMessage(Model): - """ - 内部消息 - """ __tablename__ = "common_internal_message" - title = db.Column(db.VARCHAR(255), nullable=True, comment='标题') - content = db.Column(db.TEXT, nullable=True, comment='内容') - path = db.Column(db.VARCHAR(255), nullable=True, comment='跳转路径') - is_read = db.Column(db.Boolean, default=False, comment='是否已读') - app_name = db.Column(db.VARCHAR(128), nullable=False, comment='应用名称') - category = db.Column(db.VARCHAR(128), nullable=False, comment='分类') - message_data = db.Column(db.JSON, nullable=True, comment='数据') + title = db.Column(db.VARCHAR(255), nullable=True) + content = db.Column(db.TEXT, nullable=True) + path = db.Column(db.VARCHAR(255), nullable=True) + is_read = db.Column(db.Boolean, default=False) + app_name = db.Column(db.VARCHAR(128), nullable=False) + category = db.Column(db.VARCHAR(128), nullable=False) + message_data = db.Column(db.JSON, nullable=True) employee_id = db.Column(db.Integer, db.ForeignKey('common_employee.employee_id'), comment='ID') + + +class CommonData(Model): + __table_name__ = 'common_data' + + data_type = db.Column(db.VARCHAR(255), default='') + data = db.Column(db.JSON) + + +class NoticeConfig(Model): + __tablename__ = "common_notice_config" + + platform = db.Column(db.VARCHAR(255), nullable=False) + info = db.Column(db.JSON) diff --git a/cmdb-api/api/resource.py b/cmdb-api/api/resource.py index 3bed92b5..1fc79f55 100644 --- a/cmdb-api/api/resource.py +++ b/cmdb-api/api/resource.py @@ -2,7 +2,8 @@ import os import sys -from inspect import getmembers, isclass +from inspect import getmembers +from inspect import isclass import six from flask import jsonify @@ -27,16 +28,15 @@ def send_file(*args, **kwargs): return send_file(*args, **kwargs) -API_PACKAGE = "api" +API_PACKAGE = os.path.abspath(os.path.dirname(__file__)) def register_resources(resource_path, rest_api): for root, _, files in os.walk(os.path.join(resource_path)): for filename in files: if not filename.startswith("_") and filename.endswith("py"): - module_path = os.path.join(API_PACKAGE, root[root.index("views"):]) - if module_path not in sys.path: - sys.path.insert(1, module_path) + if root not in sys.path: + sys.path.insert(1, root) view = __import__(os.path.splitext(filename)[0]) resource_list = [o[0] for o in getmembers(view) if isclass(o[1]) and issubclass(o[1], Resource)] resource_list = [i for i in resource_list if i != "APIView"] @@ -46,5 +46,4 @@ def register_resources(resource_path, rest_api): resource_cls.url_prefix = ("",) if isinstance(resource_cls.url_prefix, six.string_types): resource_cls.url_prefix = (resource_cls.url_prefix,) - rest_api.add_resource(resource_cls, *resource_cls.url_prefix) diff --git a/cmdb-api/api/tasks/acl.py b/cmdb-api/api/tasks/acl.py index e083ffe6..750eb2ef 100644 --- a/cmdb-api/api/tasks/acl.py +++ b/cmdb-api/api/tasks/acl.py @@ -5,17 +5,21 @@ from celery_once import QueueOnce from flask import current_app -from werkzeug.exceptions import BadRequest, NotFound +from werkzeug.exceptions import BadRequest +from werkzeug.exceptions import NotFound from api.extensions import celery -from api.extensions import db +from api.lib.decorator import flush_db +from api.lib.decorator import reconnect_db +from api.lib.perm.acl.audit import AuditCRUD +from api.lib.perm.acl.audit import AuditOperateSource +from api.lib.perm.acl.audit import AuditOperateType from api.lib.perm.acl.cache import AppCache from api.lib.perm.acl.cache import RoleCache from api.lib.perm.acl.cache import RoleRelationCache from api.lib.perm.acl.cache import UserCache from api.lib.perm.acl.const import ACL_QUEUE from api.lib.perm.acl.record import OperateRecordCRUD -from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType, AuditOperateSource from api.models.acl import Resource from api.models.acl import Role from api.models.acl import Trigger @@ -25,6 +29,7 @@ name="acl.role_rebuild", queue=ACL_QUEUE, once={"graceful": True, "unlock_before_run": True}) +@reconnect_db def role_rebuild(rids, app_id): rids = rids if isinstance(rids, list) else [rids] for rid in rids: @@ -34,6 +39,7 @@ def role_rebuild(rids, app_id): @celery.task(name="acl.update_resource_to_build_role", queue=ACL_QUEUE) +@reconnect_db def update_resource_to_build_role(resource_id, app_id, group_id=None): rids = [i.id for i in Role.get_by(__func_isnot__key_uid=None, fl='id', to_dict=False)] rids += [i.id for i in Role.get_by(app_id=app_id, fl='id', to_dict=False)] @@ -49,9 +55,9 @@ def update_resource_to_build_role(resource_id, app_id, group_id=None): @celery.task(name="acl.apply_trigger", queue=ACL_QUEUE) +@flush_db +@reconnect_db def apply_trigger(_id, resource_id=None, operator_uid=None): - db.session.remove() - from api.lib.perm.acl.permission import PermissionCRUD trigger = Trigger.get_by_id(_id) @@ -115,9 +121,9 @@ def apply_trigger(_id, resource_id=None, operator_uid=None): @celery.task(name="acl.cancel_trigger", queue=ACL_QUEUE) +@flush_db +@reconnect_db def cancel_trigger(_id, resource_id=None, operator_uid=None): - db.session.remove() - from api.lib.perm.acl.permission import PermissionCRUD trigger = Trigger.get_by_id(_id) @@ -183,6 +189,7 @@ def cancel_trigger(_id, resource_id=None, operator_uid=None): @celery.task(name="acl.op_record", queue=ACL_QUEUE) +@reconnect_db def op_record(app, rolename, operate_type, obj): if isinstance(app, int): app = AppCache.get(app) diff --git a/cmdb-api/api/tasks/cmdb.py b/cmdb-api/api/tasks/cmdb.py index 4d120ed9..c41f5560 100644 --- a/cmdb-api/api/tasks/cmdb.py +++ b/cmdb-api/api/tasks/cmdb.py @@ -4,9 +4,8 @@ import json import time -import jinja2 -import requests from flask import current_app +from flask_login import login_user import api.lib.cmdb.ci from api.extensions import celery @@ -17,15 +16,23 @@ from api.lib.cmdb.const import CMDB_QUEUE from api.lib.cmdb.const import REDIS_PREFIX_CI from api.lib.cmdb.const import REDIS_PREFIX_CI_RELATION -from api.lib.mail import send_mail +from api.lib.decorator import flush_db +from api.lib.decorator import reconnect_db +from api.lib.perm.acl.cache import UserCache from api.lib.utils import Lock +from api.lib.utils import handle_arg_list +from api.models.cmdb import CI from api.models.cmdb import CIRelation +from api.models.cmdb import CITypeAttribute @celery.task(name="cmdb.ci_cache", queue=CMDB_QUEUE) -def ci_cache(ci_id): +@flush_db +@reconnect_db +def ci_cache(ci_id, operate_type, record_id): + from api.lib.cmdb.ci import CITriggerManager + time.sleep(0.01) - db.session.remove() m = api.lib.cmdb.ci.CIManager() ci_dict = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False) @@ -37,11 +44,18 @@ def ci_cache(ci_id): current_app.logger.info("{0} flush..........".format(ci_id)) + if operate_type: + current_app.test_request_context().push() + login_user(UserCache.get('worker')) + + CITriggerManager.fire(operate_type, ci_dict, record_id) + @celery.task(name="cmdb.batch_ci_cache", queue=CMDB_QUEUE) -def batch_ci_cache(ci_ids): +@flush_db +@reconnect_db +def batch_ci_cache(ci_ids, ): # only for attribute change index time.sleep(1) - db.session.remove() for ci_id in ci_ids: m = api.lib.cmdb.ci.CIManager() @@ -56,6 +70,7 @@ def batch_ci_cache(ci_ids): @celery.task(name="cmdb.ci_delete", queue=CMDB_QUEUE) +@reconnect_db def ci_delete(ci_id): current_app.logger.info(ci_id) @@ -67,10 +82,22 @@ def ci_delete(ci_id): current_app.logger.info("{0} delete..........".format(ci_id)) +@celery.task(name="cmdb.ci_delete_trigger", queue=CMDB_QUEUE) +@reconnect_db +def ci_delete_trigger(trigger, operate_type, ci_dict): + current_app.logger.info('delete ci {} trigger'.format(ci_dict['_id'])) + from api.lib.cmdb.ci import CITriggerManager + + current_app.test_request_context().push() + login_user(UserCache.get('worker')) + + CITriggerManager.fire_by_trigger(trigger, operate_type, ci_dict) + + @celery.task(name="cmdb.ci_relation_cache", queue=CMDB_QUEUE) +@flush_db +@reconnect_db def ci_relation_cache(parent_id, child_id): - db.session.remove() - with Lock("CIRelation_{}".format(parent_id)): children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0] children = json.loads(children) if children is not None else {} @@ -84,7 +111,56 @@ def ci_relation_cache(parent_id, child_id): current_app.logger.info("ADD ci relation cache: {0} -> {1}".format(parent_id, child_id)) +@celery.task(name="cmdb.ci_relation_add", queue=CMDB_QUEUE) +@flush_db +@reconnect_db +def ci_relation_add(parent_dict, child_id, uid): + """ + :param parent_dict: key is '$parent_model.attr_name' + :param child_id: + :param uid: + :return: + """ + from api.lib.cmdb.ci import CIRelationManager + from api.lib.cmdb.ci_type import CITypeAttributeManager + from api.lib.cmdb.search import SearchError + from api.lib.cmdb.search.ci import search + + current_app.test_request_context().push() + login_user(UserCache.get(uid)) + + for parent in parent_dict: + parent_ci_type_name, _attr_name = parent.strip()[1:].split('.', 1) + attr_name = CITypeAttributeManager.get_attr_name(parent_ci_type_name, _attr_name) + if attr_name is None: + current_app.logger.warning("attr name {} does not exist".format(_attr_name)) + continue + + parent_dict[parent] = handle_arg_list(parent_dict[parent]) + for v in parent_dict[parent]: + query = "_type:{},{}:{}".format(parent_ci_type_name, attr_name, v) + s = search(query) + try: + response, _, _, _, _, _ = s.search() + except SearchError as e: + current_app.logger.error('ci relation add failed: {}'.format(e)) + continue + + for ci in response: + try: + CIRelationManager.add(ci['_id'], child_id) + ci_relation_cache(ci['_id'], child_id) + except Exception as e: + current_app.logger.warning(e) + finally: + try: + db.session.commit() + except: + pass + + @celery.task(name="cmdb.ci_relation_delete", queue=CMDB_QUEUE) +@reconnect_db def ci_relation_delete(parent_id, child_id): with Lock("CIRelation_{}".format(parent_id)): children = rd.get([parent_id], REDIS_PREFIX_CI_RELATION)[0] @@ -99,15 +175,19 @@ def ci_relation_delete(parent_id, child_id): @celery.task(name="cmdb.ci_type_attribute_order_rebuild", queue=CMDB_QUEUE) -def ci_type_attribute_order_rebuild(type_id): +@flush_db +@reconnect_db +def ci_type_attribute_order_rebuild(type_id, uid): current_app.logger.info('rebuild attribute order') - db.session.remove() from api.lib.cmdb.ci_type import CITypeAttributeGroupManager attrs = CITypeAttributesCache.get(type_id) id2attr = {attr.attr_id: attr for attr in attrs} + current_app.test_request_context().push() + login_user(UserCache.get(uid)) + res = CITypeAttributeGroupManager.get_by_type_id(type_id, True) order = 0 for group in res: @@ -118,41 +198,17 @@ def ci_type_attribute_order_rebuild(type_id): order += 1 -@celery.task(name='cmdb.trigger_notify', queue=CMDB_QUEUE) -def trigger_notify(notify, ci_id): - from api.lib.perm.acl.cache import UserCache - - def _wrap_mail(mail_to): - if "@" not in mail_to: - user = UserCache.get(mail_to) - if user: - return user.email +@celery.task(name="cmdb.calc_computed_attribute", queue=CMDB_QUEUE) +@flush_db +@reconnect_db +def calc_computed_attribute(attr_id, uid): + from api.lib.cmdb.ci import CIManager - return mail_to - - db.session.remove() - - m = api.lib.cmdb.ci.CIManager() - ci_dict = m.get_ci_by_id_from_db(ci_id, need_children=False, use_master=False) + current_app.test_request_context().push() + login_user(UserCache.get(uid)) - subject = jinja2.Template(notify.get('subject') or "").render(ci_dict) - body = jinja2.Template(notify.get('body') or "").render(ci_dict) - - if notify.get('wx_to'): - to_user = jinja2.Template('|'.join(notify['wx_to'])).render(ci_dict) - url = current_app.config.get("WX_URI") - data = {"to_user": to_user, "content": subject} - try: - requests.post(url, data=data) - except Exception as e: - current_app.logger.error(str(e)) - - if notify.get('mail_to'): - try: - if len(subject) > 700: - subject = subject[:600] + "..." + subject[-100:] - - send_mail("", [_wrap_mail(jinja2.Template(i).render(ci_dict)) - for i in notify['mail_to'] if i], subject, body) - except Exception as e: - current_app.logger.error("Send mail failed: {0}".format(str(e))) + cim = CIManager() + for i in CITypeAttribute.get_by(attr_id=attr_id, to_dict=False): + cis = CI.get_by(type_id=i.type_id, to_dict=False) + for ci in cis: + cim.update(ci.id, {}) diff --git a/cmdb-api/api/tasks/common_setting.py b/cmdb-api/api/tasks/common_setting.py index 45942189..ca0f6692 100644 --- a/cmdb-api/api/tasks/common_setting.py +++ b/cmdb-api/api/tasks/common_setting.py @@ -13,13 +13,9 @@ @celery.task(name="common_setting.edit_employee_department_in_acl", queue=COMMON_SETTING_QUEUE) def edit_employee_department_in_acl(e_list, new_d_id, op_uid): """ - 在 ACL 员工更换部门 - :param e_list: 员工列表 {acl_rid: 11, department_id: 22} - :param new_d_id: 新部门 ID - :param op_uid: 操作人 ID - - 在老部门中删除员工 - 在新部门中添加员工 + :param e_list:{acl_rid: 11, department_id: 22} + :param new_d_id + :param op_uid """ db.session.remove() @@ -43,7 +39,6 @@ def edit_employee_department_in_acl(e_list, new_d_id, op_uid): new_department_acl_rid = new_department.acl_rid if new_d_rid_in_acl == new_department.acl_rid else new_d_rid_in_acl for employee in e_list: - # 根据 部门ID获取部门 acl_rid old_department = Department.get_by( first=True, department_id=employee.get('department_id'), to_dict=False) if not old_department: @@ -61,7 +56,6 @@ def edit_employee_department_in_acl(e_list, new_d_id, op_uid): acl_rid=old_d_rid_in_acl ) d_acl_rid = old_department.acl_rid if old_d_rid_in_acl == old_department.acl_rid else old_d_rid_in_acl - # 在老部门中删除员工 payload = { 'app_id': 'acl', 'parent_id': d_acl_rid, @@ -71,7 +65,6 @@ def edit_employee_department_in_acl(e_list, new_d_id, op_uid): except Exception as e: result.append(ErrFormat.acl_remove_user_from_role_failed.format(str(e))) - # 在新部门中添加员工 payload = { 'app_id': 'acl', 'child_ids': [employee_acl_rid], diff --git a/cmdb-api/api/views/account.py b/cmdb-api/api/views/account.py index e2740c14..2b8d0016 100644 --- a/cmdb-api/api/views/account.py +++ b/cmdb-api/api/views/account.py @@ -2,12 +2,13 @@ import datetime -import six import jwt +import six from flask import abort from flask import current_app from flask import request -from flask_login import login_user, logout_user +from flask_login import login_user +from flask_login import logout_user from api.lib.decorator import args_required from api.lib.perm.acl.cache import User diff --git a/cmdb-api/api/views/acl/resources.py b/cmdb-api/api/views/acl/resources.py index da5fc8af..4a15a0bf 100644 --- a/cmdb-api/api/views/acl/resources.py +++ b/cmdb-api/api/views/acl/resources.py @@ -1,7 +1,7 @@ # -*- coding:utf-8 -*- -from flask import g from flask import request +from flask_login import current_user from api.lib.decorator import args_required from api.lib.decorator import args_validate @@ -103,8 +103,8 @@ def post(self): type_id = request.values.get('type_id') app_id = request.values.get('app_id') uid = request.values.get('uid') - if not uid and hasattr(g, "user") and hasattr(g.user, "uid"): - uid = g.user.uid + if not uid and hasattr(current_user, "uid"): + uid = current_user.uid resource = ResourceCRUD.add(name, type_id, app_id, uid) diff --git a/cmdb-api/api/views/acl/role.py b/cmdb-api/api/views/acl/role.py index 1afad374..03a45e15 100644 --- a/cmdb-api/api/views/acl/role.py +++ b/cmdb-api/api/views/acl/role.py @@ -2,8 +2,8 @@ from flask import abort from flask import current_app -from flask import g from flask import request +from flask_login import current_user from api.lib.decorator import args_required from api.lib.decorator import args_validate @@ -31,12 +31,9 @@ def get(self): page_size = get_page_size(request.values.get("page_size")) q = request.values.get('q') app_id = request.values.get('app_id') - is_all = request.values.get('is_all', True) - is_all = True if is_all in current_app.config.get("BOOL_TRUE") else False - user_role = request.values.get('user_role', True) - user_only = request.values.get('user_only', False) - user_role = True if user_role in current_app.config.get("BOOL_TRUE") else False - user_only = True if user_only in current_app.config.get("BOOL_TRUE") else False + is_all = request.values.get('is_all', True) in current_app.config.get("BOOL_TRUE") + user_role = request.values.get('user_role', True) in current_app.config.get("BOOL_TRUE") + user_only = request.values.get('user_only', False) in current_app.config.get("BOOL_TRUE") numfound, roles = RoleCRUD.search(q, app_id, page, page_size, user_role, is_all, user_only) @@ -160,8 +157,8 @@ class RoleHasPermissionView(APIView): @auth_with_app_token def get(self): if not request.values.get('rid'): - role = RoleCache.get_by_name(None, g.user.username) - role or abort(404, ErrFormat.role_not_found.format(g.user.username)) + role = RoleCache.get_by_name(None, current_user.username) + role or abort(404, ErrFormat.role_not_found.format(current_user.username)) else: role = RoleCache.get(int(request.values.get('rid'))) diff --git a/cmdb-api/api/views/acl/user.py b/cmdb-api/api/views/acl/user.py index dc681a4f..fcf8a6a9 100644 --- a/cmdb-api/api/views/acl/user.py +++ b/cmdb-api/api/views/acl/user.py @@ -4,7 +4,6 @@ import requests from flask import abort from flask import current_app -from flask import g from flask import request from flask import session from flask_login import current_user @@ -13,7 +12,6 @@ from api.lib.decorator import args_validate from api.lib.perm.acl.acl import ACLManager from api.lib.perm.acl.acl import role_required -from api.lib.perm.acl.audit import AuditCRUD, AuditOperateType from api.lib.perm.acl.cache import AppCache from api.lib.perm.acl.cache import UserCache from api.lib.perm.acl.resp_format import ErrFormat @@ -116,7 +114,7 @@ def put(self, uid): @role_required("acl_admin") def delete(self, uid): - if g.user.uid == uid: + if current_user.uid == uid: return abort(400, ErrFormat.invalid_operation) UserCRUD.delete(uid) @@ -162,8 +160,8 @@ def post(self): if app.name not in ('cas-server', 'acl'): return abort(403, ErrFormat.invalid_request) - elif hasattr(g, 'user'): - if g.user.username != request.values['username']: + elif hasattr(current_user, 'username'): + if current_user.username != request.values['username']: return abort(403, ErrFormat.invalid_request) else: diff --git a/cmdb-api/api/views/cmdb/attribute.py b/cmdb-api/api/views/cmdb/attribute.py index 08864f1c..fffbcdce 100644 --- a/cmdb-api/api/views/cmdb/attribute.py +++ b/cmdb-api/api/views/cmdb/attribute.py @@ -33,7 +33,8 @@ def get(self): class AttributeView(APIView): - url_prefix = ("/attributes", "/attributes/", "/attributes/") + url_prefix = ("/attributes", "/attributes/", "/attributes/", + "/attributes//calc_computed_attribute") def get(self, attr_name=None, attr_id=None): attr_manager = AttributeManager() @@ -63,17 +64,25 @@ def post(self): current_app.logger.debug(params) attr_id = AttributeManager.add(**params) + return self.jsonify(attr_id=attr_id) @args_validate(AttributeManager.cls) def put(self, attr_id): + if request.url.endswith("/calc_computed_attribute"): + AttributeManager.calc_computed_attribute(attr_id) + + return self.jsonify(attr_id=attr_id) + choice_value = handle_arg_list(request.values.get("choice_value")) params = request.values params["choice_value"] = choice_value current_app.logger.debug(params) AttributeManager().update(attr_id, **params) + return self.jsonify(attr_id=attr_id) def delete(self, attr_id): attr_name = AttributeManager.delete(attr_id) + return self.jsonify(message="attribute {0} deleted".format(attr_name)) diff --git a/cmdb-api/api/views/cmdb/auto_discovery.py b/cmdb-api/api/views/cmdb/auto_discovery.py index 2a4e6009..958d8313 100644 --- a/cmdb-api/api/views/cmdb/auto_discovery.py +++ b/cmdb-api/api/views/cmdb/auto_discovery.py @@ -5,8 +5,8 @@ from flask import abort from flask import current_app -from flask import g from flask import request +from flask_login import current_user from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCICRUD from api.lib.cmdb.auto_discovery.auto_discovery import AutoDiscoveryCITypeCRUD @@ -75,9 +75,9 @@ def get(self): # export return self.send_file(bf, as_attachment=True, - attachment_filename="cmdb_auto_discovery.json", + download_name="cmdb_auto_discovery.json", mimetype='application/json', - cache_timeout=0) + max_age=0) def post(self): f = request.files.get('file') @@ -119,7 +119,7 @@ def get(self, type_id): _, res = AutoDiscoveryCITypeCRUD.search(page=1, page_size=100000, type_id=type_id, **request.values) for i in res: if isinstance(i.get("extra_option"), dict) and i['extra_option'].get('secret'): - if not (g.user.username == "cmdb_agent" or g.user.uid == i['uid']): + if not (current_user.username == "cmdb_agent" or current_user.uid == i['uid']): i['extra_option'].pop('secret', None) else: i['extra_option']['secret'] = AESCrypto.decrypt(i['extra_option']['secret']) @@ -213,7 +213,7 @@ class AutoDiscoveryRuleSyncView(APIView): url_prefix = ("/adt/sync",) def get(self): - if g.user.username not in ("cmdb_agent", "worker", "admin"): + if current_user.username not in ("cmdb_agent", "worker", "admin"): return abort(403) oneagent_name = request.values.get('oneagent_name') diff --git a/cmdb-api/api/views/cmdb/ci.py b/cmdb-api/api/views/cmdb/ci.py index cb285b59..ce39962a 100644 --- a/cmdb-api/api/views/cmdb/ci.py +++ b/cmdb-api/api/views/cmdb/ci.py @@ -11,7 +11,8 @@ from api.lib.cmdb.ci import CIManager from api.lib.cmdb.ci import CIRelationManager from api.lib.cmdb.const import ExistPolicy -from api.lib.cmdb.const import ResourceTypeEnum, PermEnum +from api.lib.cmdb.const import PermEnum +from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.const import RetKey from api.lib.cmdb.perms import has_perm_for_ci from api.lib.cmdb.search import SearchError @@ -83,11 +84,10 @@ def post(self): ci_dict = self._wrap_ci_dict() manager = CIManager() - current_app.logger.debug(ci_dict) ci_id = manager.add(ci_type, exist_policy=exist_policy or ExistPolicy.REJECT, _no_attribute_policy=_no_attribute_policy, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) return self.jsonify(ci_id=ci_id) @@ -95,7 +95,6 @@ def post(self): @has_perm_for_ci("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type) def put(self, ci_id=None): args = request.values - current_app.logger.info(args) ci_type = args.get("ci_type") _no_attribute_policy = args.get("no_attribute_policy", ExistPolicy.IGNORE) @@ -103,13 +102,14 @@ def put(self, ci_id=None): manager = CIManager() if ci_id is not None: manager.update(ci_id, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) else: + request.values.pop('exist_policy', None) ci_id = manager.add(ci_type, exist_policy=ExistPolicy.REPLACE, _no_attribute_policy=_no_attribute_policy, - _is_admin=request.values.pop('__is_admin', False), + _is_admin=request.values.pop('__is_admin', None) or False, **ci_dict) return self.jsonify(ci_id=ci_id) @@ -183,8 +183,8 @@ class CIUnique(APIView): @has_perm_from_args("ci_id", ResourceTypeEnum.CI, PermEnum.UPDATE, CIManager.get_type_name) def put(self, ci_id): params = request.values - unique_name = params.keys()[0] - unique_value = params.values()[0] + unique_name = list(params.keys())[0] + unique_value = list(params.values())[0] CIManager.update_unique_value(ci_id, unique_name, unique_value) @@ -226,11 +226,11 @@ def get(self, ci_id=None): from api.tasks.cmdb import ci_cache from api.lib.cmdb.const import CMDB_QUEUE if ci_id is not None: - ci_cache.apply_async([ci_id], queue=CMDB_QUEUE) + ci_cache.apply_async(args=(ci_id, None, None), queue=CMDB_QUEUE) else: cis = CI.get_by(to_dict=False) for ci in cis: - ci_cache.apply_async([ci.id], queue=CMDB_QUEUE) + ci_cache.apply_async(args=(ci.id, None, None), queue=CMDB_QUEUE) return self.jsonify(code=200) @@ -240,3 +240,13 @@ class CIAutoDiscoveryStatisticsView(APIView): def get(self): return self.jsonify(CIManager.get_ad_statistics()) + + +class CIPasswordView(APIView): + url_prefix = "/ci//attributes//password" + + def get(self, ci_id, attr_id): + return self.jsonify(ci_id=ci_id, attr_id=attr_id, value=CIManager.load_password(ci_id, attr_id)) + + def post(self, ci_id, attr_id): + return self.get(ci_id, attr_id) diff --git a/cmdb-api/api/views/cmdb/ci_type.py b/cmdb-api/api/views/cmdb/ci_type.py index 7d748e16..4e02d40d 100644 --- a/cmdb-api/api/views/cmdb/ci_type.py +++ b/cmdb-api/api/views/cmdb/ci_type.py @@ -1,4 +1,4 @@ -# -*- coding:utf-8 -*- +# -*- coding:utf-8 -*- import json @@ -154,9 +154,15 @@ def post(self, type_id): class CITypeAttributeView(APIView): - url_prefix = ("/ci_types//attributes", "/ci_types//attributes") + url_prefix = ("/ci_types//attributes", "/ci_types//attributes", + "/ci_types/common_attributes") def get(self, type_id=None, type_name=None): + if request.path.endswith("/common_attributes"): + type_ids = handle_arg_list(request.values.get('type_ids')) + + return self.jsonify(attributes=CITypeAttributeManager.get_common_attributes(type_ids)) + t = CITypeCache.get(type_id) or CITypeCache.get(type_name) or abort(404, ErrFormat.ci_type_not_found) type_id = t.id unique_id = t.unique_id @@ -350,9 +356,9 @@ def get(self): # export return self.send_file(bf, as_attachment=True, - attachment_filename="cmdb_template.json", + download_name="cmdb_template.json", mimetype='application/json', - cache_timeout=0) + max_age=0) @role_required(RoleEnum.CONFIG) def post(self): # import @@ -413,22 +419,22 @@ def get(self, type_id): return self.jsonify(CITypeTriggerManager.get(type_id)) @has_perm_from_args("type_id", ResourceTypeEnum.CI, PermEnum.CONFIG, CITypeManager.get_name_by_id) - @args_required("attr_id") - @args_required("notify") + @args_required("option") def post(self, type_id): - attr_id = request.values.get('attr_id') - notify = request.values.get('notify') + attr_id = request.values.get('attr_id') or None + option = request.values.get('option') - return self.jsonify(CITypeTriggerManager().add(type_id, attr_id, notify)) + return self.jsonify(CITypeTriggerManager().add(type_id, attr_id, option)) @has_perm_from_args("type_id", ResourceTypeEnum.CI, PermEnum.CONFIG, CITypeManager.get_name_by_id) - @args_required("notify") + @args_required("option") def put(self, type_id, _id): assert type_id is not None - notify = request.values.get('notify') + option = request.values.get('option') + attr_id = request.values.get('attr_id') - return self.jsonify(CITypeTriggerManager().update(_id, notify)) + return self.jsonify(CITypeTriggerManager().update(_id, attr_id, option)) @has_perm_from_args("type_id", ResourceTypeEnum.CI, PermEnum.CONFIG, CITypeManager.get_name_by_id) def delete(self, type_id, _id): diff --git a/cmdb-api/api/views/cmdb/ci_type_relation.py b/cmdb-api/api/views/cmdb/ci_type_relation.py index a78889eb..20ce3231 100644 --- a/cmdb-api/api/views/cmdb/ci_type_relation.py +++ b/cmdb-api/api/views/cmdb/ci_type_relation.py @@ -6,7 +6,9 @@ from api.lib.cmdb.ci_type import CITypeManager from api.lib.cmdb.ci_type import CITypeRelationManager -from api.lib.cmdb.const import PermEnum, ResourceTypeEnum, RoleEnum +from api.lib.cmdb.const import PermEnum +from api.lib.cmdb.const import ResourceTypeEnum +from api.lib.cmdb.const import RoleEnum from api.lib.cmdb.resp_format import ErrFormat from api.lib.decorator import args_required from api.lib.perm.acl.acl import ACLManager @@ -17,9 +19,14 @@ class GetChildrenView(APIView): - url_prefix = "/ci_type_relations//children" + url_prefix = ("/ci_type_relations//children", + "/ci_type_relations//recursive_level2children", + ) def get(self, parent_id): + if request.url.endswith("recursive_level2children"): + return self.jsonify(CITypeRelationManager.recursive_level2children(parent_id)) + return self.jsonify(children=CITypeRelationManager.get_children(parent_id)) diff --git a/cmdb-api/api/views/cmdb/custom_dashboard.py b/cmdb-api/api/views/cmdb/custom_dashboard.py index 4991ceb7..1c20ff93 100644 --- a/cmdb-api/api/views/cmdb/custom_dashboard.py +++ b/cmdb-api/api/views/cmdb/custom_dashboard.py @@ -13,7 +13,8 @@ class CustomDashboardApiView(APIView): - url_prefix = ("/custom_dashboard", "/custom_dashboard/", "/custom_dashboard/batch") + url_prefix = ("/custom_dashboard", "/custom_dashboard/", "/custom_dashboard/batch", + "/custom_dashboard/preview") def get(self): return self.jsonify(CustomDashboardManager.get()) @@ -21,17 +22,26 @@ def get(self): @role_required(RoleEnum.CONFIG) @args_validate(CustomDashboardManager.cls) def post(self): - cm = CustomDashboardManager.add(**request.values) + if request.url.endswith("/preview"): + return self.jsonify(counter=CustomDashboardManager.preview(**request.values)) - return self.jsonify(cm.to_dict()) + cm, counter = CustomDashboardManager.add(**request.values) + + res = cm.to_dict() + res.update(counter=counter) + + return self.jsonify(res) @role_required(RoleEnum.CONFIG) @args_validate(CustomDashboardManager.cls) def put(self, _id=None): if _id is not None: - cm = CustomDashboardManager.update(_id, **request.values) + cm, counter = CustomDashboardManager.update(_id, **request.values) + + res = cm.to_dict() + res.update(counter=counter) - return self.jsonify(cm.to_dict()) + return self.jsonify(res) CustomDashboardManager.batch_update(request.values.get("id2options")) diff --git a/cmdb-api/api/views/cmdb/history.py b/cmdb-api/api/views/cmdb/history.py index 867d18ac..ceaa2dbf 100644 --- a/cmdb-api/api/views/cmdb/history.py +++ b/cmdb-api/api/views/cmdb/history.py @@ -5,14 +5,18 @@ from flask import abort from flask import request +from flask import session from api.lib.cmdb.ci import CIManager -from api.lib.cmdb.const import ResourceTypeEnum, PermEnum +from api.lib.cmdb.const import PermEnum +from api.lib.cmdb.const import ResourceTypeEnum from api.lib.cmdb.const import RoleEnum from api.lib.cmdb.history import AttributeHistoryManger +from api.lib.cmdb.history import CITriggerHistoryManager from api.lib.cmdb.history import CITypeHistoryManager from api.lib.cmdb.resp_format import ErrFormat from api.lib.perm.acl.acl import has_perm_from_args +from api.lib.perm.acl.acl import is_app_admin from api.lib.perm.acl.acl import role_required from api.lib.utils import get_page from api.lib.utils import get_page_size @@ -75,6 +79,39 @@ def get(self, ci_id): return self.jsonify(result) +class CITriggerHistoryView(APIView): + url_prefix = ("/history/ci_triggers/", "/history/ci_triggers") + + @has_perm_from_args("ci_id", ResourceTypeEnum.CI, PermEnum.READ, CIManager.get_type_name) + def get(self, ci_id=None): + if ci_id is not None: + result = CITriggerHistoryManager.get_by_ci_id(ci_id) + + return self.jsonify(result) + + if RoleEnum.CONFIG not in session.get("acl", {}).get("parentRoles", []) and not is_app_admin("cmdb"): + return abort(403, ErrFormat.role_required.format(RoleEnum.CONFIG)) + + type_id = request.values.get("type_id") + trigger_id = request.values.get("trigger_id") + operate_type = request.values.get("operate_type") + + page = get_page(request.values.get('page', 1)) + page_size = get_page_size(request.values.get('page_size', 1)) + + numfound, result = CITriggerHistoryManager.get(page, + page_size, + type_id=type_id, + trigger_id=trigger_id, + operate_type=operate_type) + + return self.jsonify(page=page, + page_size=page_size, + numfound=numfound, + total=len(result), + result=result) + + class CITypeHistoryView(APIView): url_prefix = "/history/ci_types" diff --git a/cmdb-api/api/views/cmdb/inner_secrets.py b/cmdb-api/api/views/cmdb/inner_secrets.py new file mode 100644 index 00000000..573ededa --- /dev/null +++ b/cmdb-api/api/views/cmdb/inner_secrets.py @@ -0,0 +1,37 @@ +from flask import request + +from api.lib.perm.auth import auth_abandoned +from api.lib.secrets.inner import KeyManage +from api.lib.secrets.secrets import InnerKVManger +from api.resource import APIView + + +class InnerSecretUnSealView(APIView): + url_prefix = "/secrets/unseal" + + @auth_abandoned + def post(self): + unseal_key = request.headers.get("Unseal-Token") + res = KeyManage(backend=InnerKVManger()).unseal(unseal_key) + return self.jsonify(**res) + + +class InnerSecretSealView(APIView): + url_prefix = "/secrets/seal" + + @auth_abandoned + def post(self): + unseal_key = request.headers.get("Inner-Token") + res = KeyManage(backend=InnerKVManger()).seal(unseal_key) + return self.jsonify(**res) + + +class InnerSecretAutoSealView(APIView): + url_prefix = "/secrets/auto_seal" + + @auth_abandoned + def post(self): + root_key = request.headers.get("Inner-Token") + res = KeyManage(trigger=root_key, + backend=InnerKVManger()).auto_unseal() + return self.jsonify(**res) diff --git a/cmdb-api/api/views/cmdb/preference.py b/cmdb-api/api/views/cmdb/preference.py index 67846941..cc3dab6d 100644 --- a/cmdb-api/api/views/cmdb/preference.py +++ b/cmdb-api/api/views/cmdb/preference.py @@ -5,7 +5,9 @@ from flask import request from api.lib.cmdb.ci_type import CITypeManager -from api.lib.cmdb.const import PermEnum, ResourceTypeEnum, RoleEnum +from api.lib.cmdb.const import PermEnum +from api.lib.cmdb.const import ResourceTypeEnum +from api.lib.cmdb.const import RoleEnum from api.lib.cmdb.perms import CIFilterPermsCRUD from api.lib.cmdb.preference import PreferenceManager from api.lib.cmdb.resp_format import ErrFormat diff --git a/cmdb-api/api/views/common_setting/common_data.py b/cmdb-api/api/views/common_setting/common_data.py new file mode 100644 index 00000000..3793a5e8 --- /dev/null +++ b/cmdb-api/api/views/common_setting/common_data.py @@ -0,0 +1,35 @@ +from flask import request + +from api.lib.common_setting.common_data import CommonDataCRUD +from api.resource import APIView + +prefix = '/data' + + +class DataView(APIView): + url_prefix = (f'{prefix}/',) + + def get(self, data_type): + data_list = CommonDataCRUD.get_data_by_type(data_type) + + return self.jsonify(data_list) + + def post(self, data_type): + params = request.json + CommonDataCRUD.create_new_data(data_type, **params) + + return self.jsonify(params) + + +class DataViewWithId(APIView): + url_prefix = (f'{prefix}//',) + + def put(self, _id): + params = request.json + res = CommonDataCRUD.update_data(_id, **params) + + return self.jsonify(res.to_dict()) + + def delete(self, _id): + CommonDataCRUD.delete(_id) + return self.jsonify({}) diff --git a/cmdb-api/api/views/common_setting/company_info.py b/cmdb-api/api/views/common_setting/company_info.py index d2aca2ad..4298cfd2 100644 --- a/cmdb-api/api/views/common_setting/company_info.py +++ b/cmdb-api/api/views/common_setting/company_info.py @@ -1,9 +1,7 @@ # -*- coding:utf-8 -*- -from flask import abort from flask import request from api.lib.common_setting.company_info import CompanyInfoCRUD -from api.lib.common_setting.resp_format import ErrFormat from api.resource import APIView prefix = '/company' @@ -16,15 +14,16 @@ def get(self): return self.jsonify(CompanyInfoCRUD.get()) def post(self): - info = CompanyInfoCRUD.get() - if info: - abort(400, ErrFormat.company_info_is_already_existed) data = { 'info': { **request.values } } - d = CompanyInfoCRUD.create(**data) + info = CompanyInfoCRUD.get() + if info: + d = CompanyInfoCRUD.update(info.get('id'), **data) + else: + d = CompanyInfoCRUD.create(**data) res = d.to_dict() return self.jsonify(res) diff --git a/cmdb-api/api/views/common_setting/department.py b/cmdb-api/api/views/common_setting/department.py index f9d9c7bf..9a8dd4a7 100644 --- a/cmdb-api/api/views/common_setting/department.py +++ b/cmdb-api/api/views/common_setting/department.py @@ -100,7 +100,7 @@ class DepartmentSortView(APIView): def put(self): """ - 修改部门排序,只能在同一个上级内排序 + only can sort in the same parent """ department_list = request.json.get('department_list', None) if department_list is None: diff --git a/cmdb-api/api/views/common_setting/employee.py b/cmdb-api/api/views/common_setting/employee.py index a3c95a87..173dc132 100644 --- a/cmdb-api/api/views/common_setting/employee.py +++ b/cmdb-api/api/views/common_setting/employee.py @@ -1,7 +1,5 @@ # -*- coding:utf-8 -*- -import os - -from flask import abort, current_app, send_from_directory +from flask import abort from flask import request from werkzeug.datastructures import MultiDict @@ -146,42 +144,25 @@ def get(self): return self.jsonify(result) -class EmployeeViewExportExcel(APIView): - url_prefix = (f'{prefix}/export_all',) +class GetEmployeeNoticeByIds(APIView): + url_prefix = (f'{prefix}/get_notice_by_ids',) - def get(self): - col_desc_map = { - 'nickname': "姓名", - 'email': '邮箱', - 'sex': '性别', - 'mobile': '手机号', - 'department_name': '部门', - 'position_name': '岗位', - 'nickname_direct_supervisor': '直接上级', - 'last_login': '上次登录时间', - } - - # 规定了静态文件的存储位置 - excel_filename = 'all_employee_info.xlsx' - excel_path = current_app.config['UPLOAD_DIRECTORY_FULL'] - excel_path_with_filename = os.path.join(excel_path, excel_filename) - - # 根据parameter查表,自连接通过上级id获取上级名字列 - block_status = int(request.args.get('block_status', -1)) - df = EmployeeCRUD.get_export_employee_df(block_status) - - # 改变列名为中文head - try: - df = df.rename(columns=col_desc_map) - except Exception as e: - abort(500, ErrFormat.rename_columns_failed.format(str(e))) - - # 生成静态excel文件 - try: - df.to_excel(excel_path_with_filename, - sheet_name='Sheet1', index=False, encoding="utf-8") - except Exception as e: - current_app.logger.error(e) - abort(500, ErrFormat.generate_excel_failed.format(str(e))) - - return send_from_directory(excel_path, excel_filename, as_attachment=True) + def post(self): + employee_ids = request.json.get('employee_ids', []) + if not employee_ids: + result = [] + else: + result = EmployeeCRUD.get_employee_notice_by_ids(employee_ids) + return self.jsonify(result) + + +class EmployeeBindNoticeWithACLID(APIView): + url_prefix = (f'{prefix}/by_uid/bind_notice//',) + + def put(self, platform, _uid): + data = EmployeeCRUD.bind_notice_by_uid(platform, _uid) + return self.jsonify(info=data) + + def delete(self, platform, _uid): + data = EmployeeCRUD.remove_bind_notice_by_uid(platform, _uid) + return self.jsonify(info=data) diff --git a/cmdb-api/api/views/common_setting/file_manage.py b/cmdb-api/api/views/common_setting/file_manage.py index 23e00479..71503656 100644 --- a/cmdb-api/api/views/common_setting/file_manage.py +++ b/cmdb-api/api/views/common_setting/file_manage.py @@ -11,7 +11,7 @@ prefix = '/file' ALLOWED_EXTENSIONS = { - 'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif', 'xls', 'xlsx', 'doc', 'docx', 'ppt', 'pptx', 'csv' + 'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif', 'xls', 'xlsx', 'doc', 'docx', 'ppt', 'pptx', 'csv', 'svg' } diff --git a/cmdb-api/api/views/common_setting/notice_config.py b/cmdb-api/api/views/common_setting/notice_config.py new file mode 100644 index 00000000..e5dea630 --- /dev/null +++ b/cmdb-api/api/views/common_setting/notice_config.py @@ -0,0 +1,79 @@ +from flask import request, abort, current_app +from werkzeug.datastructures import MultiDict + +from api.lib.perm.auth import auth_with_app_token +from api.models.common_setting import NoticeConfig +from api.resource import APIView +from api.lib.common_setting.notice_config import NoticeConfigForm, NoticeConfigUpdateForm, NoticeConfigCRUD +from api.lib.decorator import args_required +from api.lib.common_setting.resp_format import ErrFormat + +prefix = '/notice_config' + + +class NoticeConfigView(APIView): + url_prefix = (f'{prefix}',) + + @args_required('platform') + @auth_with_app_token + def get(self): + platform = request.args.get('platform') + res = NoticeConfig.get_by(first=True, to_dict=True, platform=platform) or {} + return self.jsonify(res) + + def post(self): + form = NoticeConfigForm(MultiDict(request.json)) + if not form.validate(): + abort(400, ','.join(['{}: {}'.format(filed, ','.join(msg)) for filed, msg in form.errors.items()])) + + data = NoticeConfigCRUD.add_notice_config(**form.data) + return self.jsonify(data.to_dict()) + + +class NoticeConfigUpdateView(APIView): + url_prefix = (f'{prefix}/',) + + def put(self, _id): + form = NoticeConfigUpdateForm(MultiDict(request.json)) + if not form.validate(): + abort(400, ','.join(['{}: {}'.format(filed, ','.join(msg)) for filed, msg in form.errors.items()])) + + data = NoticeConfigCRUD.edit_notice_config(_id, **form.data) + return self.jsonify(data.to_dict()) + + +class CheckEmailServer(APIView): + url_prefix = (f'{prefix}/send_test_email',) + + def post(self): + receive_address = request.args.get('receive_address') + info = request.values.get('info', {}) + + try: + + result = NoticeConfigCRUD.test_send_email(receive_address, **info) + return self.jsonify(result=result) + except Exception as e: + current_app.logger.error('test_send_email err:') + current_app.logger.error(e) + if 'Timed Out' in str(e): + abort(400, ErrFormat.email_send_timeout) + abort(400, f"{str(e)}") + + +class NoticeConfigGetView(APIView): + method_decorators = [] + url_prefix = (f'{prefix}/all',) + + @auth_with_app_token + def get(self): + res = NoticeConfigCRUD.get_all() + return self.jsonify(res) + + +class NoticeAppBotView(APIView): + url_prefix = (f'{prefix}/app_bot',) + + def get(self): + res = NoticeConfigCRUD.get_app_bot() + return self.jsonify(res) diff --git a/cmdb-api/api/views/entry.py b/cmdb-api/api/views/entry.py index 8c5e43ea..9d27d23e 100644 --- a/cmdb-api/api/views/entry.py +++ b/cmdb-api/api/views/entry.py @@ -6,7 +6,9 @@ from flask_restful import Api from api.resource import register_resources -from .account import LoginView, LogoutView, AuthWithKeyView +from .account import AuthWithKeyView +from .account import LoginView +from .account import LogoutView HERE = os.path.abspath(os.path.dirname(__file__)) diff --git a/cmdb-api/autoapp.py b/cmdb-api/autoapp.py index 9ff83cc2..d16e79dd 100644 --- a/cmdb-api/autoapp.py +++ b/cmdb-api/autoapp.py @@ -1,14 +1,7 @@ # -*- coding: utf-8 -*- """Create an application instance.""" -from flask import g -from flask_login import current_user from api.app import create_app app = create_app() - - -@app.before_request -def before_request(): - g.user = current_user diff --git a/cmdb-api/celery_worker.py b/cmdb-api/celery_worker.py index 56921410..5f6dbbd9 100644 --- a/cmdb-api/celery_worker.py +++ b/cmdb-api/celery_worker.py @@ -3,7 +3,7 @@ from api.app import create_app from api.extensions import celery -# celery worker -A celery_worker.celery -l DEBUG -E -Q xxxx +# celery -A celery_worker.celery worker -l DEBUG -E -Q xxxx app = create_app() app.app_context().push() diff --git a/cmdb-api/migrations/README b/cmdb-api/migrations/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/cmdb-api/migrations/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/cmdb-api/migrations/alembic.ini b/cmdb-api/migrations/alembic.ini new file mode 100644 index 00000000..f8ed4801 --- /dev/null +++ b/cmdb-api/migrations/alembic.ini @@ -0,0 +1,45 @@ +# A generic, single database configuration. + +[alembic] +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/cmdb-api/migrations/env.py b/cmdb-api/migrations/env.py new file mode 100644 index 00000000..666f4220 --- /dev/null +++ b/cmdb-api/migrations/env.py @@ -0,0 +1,110 @@ +from __future__ import with_statement + +import logging +from logging.config import fileConfig + +from sqlalchemy import engine_from_config +from sqlalchemy import pool + +from alembic import context + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) +logger = logging.getLogger('alembic.env') + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +from flask import current_app +config.set_main_option( + 'sqlalchemy.url', current_app.config.get( + 'SQLALCHEMY_DATABASE_URI').replace('%', '%%')) +target_metadata = current_app.extensions['migrate'].db.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +# 添加要屏蔽的table列表 +exclude_tables = ["c_cfp"] + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, target_metadata=target_metadata, literal_binds=True, + include_name=include_name + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + + # this callback is used to prevent an auto-migration from being generated + # when there are no changes to the schema + # reference: http://alembic.zzzcomputing.com/en/latest/cookbook.html + def process_revision_directives(context, revision, directives): + if getattr(config.cmd_opts, 'autogenerate', False): + script = directives[0] + if script.upgrade_ops.is_empty(): + directives[:] = [] + logger.info('No changes in schema detected.') + + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix='sqlalchemy.', + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + process_revision_directives=process_revision_directives, + include_name=include_name, + **current_app.extensions['migrate'].configure_args + ) + + with context.begin_transaction(): + context.run_migrations() + + +def include_name(name, type_, parent_names): + if type_ == "table": + return name not in exclude_tables + elif parent_names.get("table_name") in exclude_tables: + return False + + return True + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/cmdb-api/migrations/script.py.mako b/cmdb-api/migrations/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/cmdb-api/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/cmdb-api/migrations/versions/6a4df2623057_.py b/cmdb-api/migrations/versions/6a4df2623057_.py new file mode 100644 index 00000000..ab6b330e --- /dev/null +++ b/cmdb-api/migrations/versions/6a4df2623057_.py @@ -0,0 +1,360 @@ +"""empty message + +Revision ID: 6a4df2623057 +Revises: +Create Date: 2023-10-13 15:17:00.066858 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +# revision identifiers, used by Alembic. +revision = '6a4df2623057' +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('common_data', + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.Boolean(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('data_type', sa.VARCHAR(length=255), nullable=True), + sa.Column('data', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_common_data_deleted'), 'common_data', ['deleted'], unique=False) + op.create_table('common_notice_config', + sa.Column('deleted_at', sa.DateTime(), nullable=True), + sa.Column('deleted', sa.Boolean(), nullable=True), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), autoincrement=True, nullable=False), + sa.Column('platform', sa.VARCHAR(length=255), nullable=False), + sa.Column('info', sa.JSON(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_common_notice_config_deleted'), 'common_notice_config', ['deleted'], unique=False) + op.add_column('c_attributes', sa.Column('choice_other', sa.JSON(), nullable=True)) + op.drop_index('idx_c_attributes_uid', table_name='c_attributes') + op.create_index(op.f('ix_c_attributes_uid'), 'c_attributes', ['uid'], unique=False) + op.drop_index('ix_c_custom_dashboard_deleted', table_name='c_c_d') + op.create_index(op.f('ix_c_c_d_deleted'), 'c_c_d', ['deleted'], unique=False) + op.drop_index('ix_c_ci_type_triggers_deleted', table_name='c_c_t_t') + op.create_index(op.f('ix_c_c_t_t_deleted'), 'c_c_t_t', ['deleted'], unique=False) + op.drop_index('ix_c_ci_type_unique_constraints_deleted', table_name='c_c_t_u_c') + op.create_index(op.f('ix_c_c_t_u_c_deleted'), 'c_c_t_u_c', ['deleted'], unique=False) + op.drop_index('c_ci_types_uid', table_name='c_ci_types') + op.create_index(op.f('ix_c_ci_types_uid'), 'c_ci_types', ['uid'], unique=False) + op.alter_column('c_prv', 'uid', + existing_type=mysql.INTEGER(), + nullable=False) + op.drop_index('ix_c_preference_relation_views_deleted', table_name='c_prv') + op.drop_index('ix_c_preference_relation_views_name', table_name='c_prv') + op.create_index(op.f('ix_c_prv_deleted'), 'c_prv', ['deleted'], unique=False) + op.create_index(op.f('ix_c_prv_name'), 'c_prv', ['name'], unique=False) + op.create_index(op.f('ix_c_prv_uid'), 'c_prv', ['uid'], unique=False) + op.drop_index('ix_c_preference_show_attributes_deleted', table_name='c_psa') + op.drop_index('ix_c_preference_show_attributes_uid', table_name='c_psa') + op.create_index(op.f('ix_c_psa_deleted'), 'c_psa', ['deleted'], unique=False) + op.create_index(op.f('ix_c_psa_uid'), 'c_psa', ['uid'], unique=False) + op.drop_index('ix_c_preference_tree_views_deleted', table_name='c_ptv') + op.drop_index('ix_c_preference_tree_views_uid', table_name='c_ptv') + op.create_index(op.f('ix_c_ptv_deleted'), 'c_ptv', ['deleted'], unique=False) + op.create_index(op.f('ix_c_ptv_uid'), 'c_ptv', ['uid'], unique=False) + op.alter_column('common_department', 'department_name', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='部门名称', + existing_nullable=True) + op.alter_column('common_department', 'department_director_id', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='部门负责人ID', + existing_nullable=True) + op.alter_column('common_department', 'department_parent_id', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='上级部门ID', + existing_nullable=True) + op.alter_column('common_department', 'sort_value', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='排序值', + existing_nullable=True) + op.alter_column('common_department', 'acl_rid', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='ACL中rid', + existing_nullable=True) + op.alter_column('common_employee', 'email', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='邮箱', + existing_nullable=True) + op.alter_column('common_employee', 'username', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='用户名', + existing_nullable=True) + op.alter_column('common_employee', 'nickname', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='姓名', + existing_nullable=True) + op.alter_column('common_employee', 'sex', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=64), + comment=None, + existing_comment='性别', + existing_nullable=True) + op.alter_column('common_employee', 'position_name', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='职位名称', + existing_nullable=True) + op.alter_column('common_employee', 'mobile', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='电话号码', + existing_nullable=True) + op.alter_column('common_employee', 'avatar', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='头像', + existing_nullable=True) + op.alter_column('common_employee', 'direct_supervisor_id', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='直接上级ID', + existing_nullable=True) + op.alter_column('common_employee', 'department_id', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='部门ID', + existing_nullable=True) + op.alter_column('common_employee', 'acl_uid', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='ACL中uid', + existing_nullable=True) + op.alter_column('common_employee', 'acl_rid', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='ACL中rid', + existing_nullable=True) + op.alter_column('common_employee', 'acl_virtual_rid', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='ACL中虚拟角色rid', + existing_nullable=True) + op.alter_column('common_employee', 'last_login', + existing_type=mysql.TIMESTAMP(), + comment=None, + existing_comment='上次登录时间', + existing_nullable=True) + op.alter_column('common_employee', 'block', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='锁定状态', + existing_nullable=True) + op.alter_column('common_employee_info', 'info', + existing_type=mysql.JSON(), + comment=None, + existing_comment='员工信息', + existing_nullable=True) + op.alter_column('common_employee_info', 'employee_id', + existing_type=mysql.INTEGER(), + comment=None, + existing_comment='员工ID', + existing_nullable=True) + op.alter_column('common_internal_message', 'title', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='标题', + existing_nullable=True) + op.alter_column('common_internal_message', 'content', + existing_type=mysql.TEXT(charset='utf8mb3', collation='utf8mb3_unicode_ci'), + comment=None, + existing_comment='内容', + existing_nullable=True) + op.alter_column('common_internal_message', 'path', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment=None, + existing_comment='跳转路径', + existing_nullable=True) + op.alter_column('common_internal_message', 'is_read', + existing_type=mysql.TINYINT(display_width=1), + comment=None, + existing_comment='是否已读', + existing_nullable=True) + op.alter_column('common_internal_message', 'app_name', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=128), + comment=None, + existing_comment='应用名称', + existing_nullable=False) + op.alter_column('common_internal_message', 'category', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=128), + comment=None, + existing_comment='分类', + existing_nullable=False) + op.alter_column('common_internal_message', 'message_data', + existing_type=mysql.JSON(), + comment=None, + existing_comment='数据', + existing_nullable=True) + op.drop_column('users', 'apps') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('users', sa.Column('apps', mysql.JSON(), nullable=True)) + op.alter_column('common_internal_message', 'message_data', + existing_type=mysql.JSON(), + comment='数据', + existing_nullable=True) + op.alter_column('common_internal_message', 'category', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=128), + comment='分类', + existing_nullable=False) + op.alter_column('common_internal_message', 'app_name', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=128), + comment='应用名称', + existing_nullable=False) + op.alter_column('common_internal_message', 'is_read', + existing_type=mysql.TINYINT(display_width=1), + comment='是否已读', + existing_nullable=True) + op.alter_column('common_internal_message', 'path', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='跳转路径', + existing_nullable=True) + op.alter_column('common_internal_message', 'content', + existing_type=mysql.TEXT(charset='utf8mb3', collation='utf8mb3_unicode_ci'), + comment='内容', + existing_nullable=True) + op.alter_column('common_internal_message', 'title', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='标题', + existing_nullable=True) + op.alter_column('common_employee_info', 'employee_id', + existing_type=mysql.INTEGER(), + comment='员工ID', + existing_nullable=True) + op.alter_column('common_employee_info', 'info', + existing_type=mysql.JSON(), + comment='员工信息', + existing_nullable=True) + op.alter_column('common_employee', 'block', + existing_type=mysql.INTEGER(), + comment='锁定状态', + existing_nullable=True) + op.alter_column('common_employee', 'last_login', + existing_type=mysql.TIMESTAMP(), + comment='上次登录时间', + existing_nullable=True) + op.alter_column('common_employee', 'acl_virtual_rid', + existing_type=mysql.INTEGER(), + comment='ACL中虚拟角色rid', + existing_nullable=True) + op.alter_column('common_employee', 'acl_rid', + existing_type=mysql.INTEGER(), + comment='ACL中rid', + existing_nullable=True) + op.alter_column('common_employee', 'acl_uid', + existing_type=mysql.INTEGER(), + comment='ACL中uid', + existing_nullable=True) + op.alter_column('common_employee', 'department_id', + existing_type=mysql.INTEGER(), + comment='部门ID', + existing_nullable=True) + op.alter_column('common_employee', 'direct_supervisor_id', + existing_type=mysql.INTEGER(), + comment='直接上级ID', + existing_nullable=True) + op.alter_column('common_employee', 'avatar', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='头像', + existing_nullable=True) + op.alter_column('common_employee', 'mobile', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='电话号码', + existing_nullable=True) + op.alter_column('common_employee', 'position_name', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='职位名称', + existing_nullable=True) + op.alter_column('common_employee', 'sex', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=64), + comment='性别', + existing_nullable=True) + op.alter_column('common_employee', 'nickname', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='姓名', + existing_nullable=True) + op.alter_column('common_employee', 'username', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='用户名', + existing_nullable=True) + op.alter_column('common_employee', 'email', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='邮箱', + existing_nullable=True) + op.alter_column('common_department', 'acl_rid', + existing_type=mysql.INTEGER(), + comment='ACL中rid', + existing_nullable=True) + op.alter_column('common_department', 'sort_value', + existing_type=mysql.INTEGER(), + comment='排序值', + existing_nullable=True) + op.alter_column('common_department', 'department_parent_id', + existing_type=mysql.INTEGER(), + comment='上级部门ID', + existing_nullable=True) + op.alter_column('common_department', 'department_director_id', + existing_type=mysql.INTEGER(), + comment='部门负责人ID', + existing_nullable=True) + op.alter_column('common_department', 'department_name', + existing_type=mysql.VARCHAR(charset='utf8mb3', collation='utf8mb3_unicode_ci', length=255), + comment='部门名称', + existing_nullable=True) + op.drop_index(op.f('ix_c_ptv_uid'), table_name='c_ptv') + op.drop_index(op.f('ix_c_ptv_deleted'), table_name='c_ptv') + op.create_index('ix_c_preference_tree_views_uid', 'c_ptv', ['uid'], unique=False) + op.create_index('ix_c_preference_tree_views_deleted', 'c_ptv', ['deleted'], unique=False) + op.drop_index(op.f('ix_c_psa_uid'), table_name='c_psa') + op.drop_index(op.f('ix_c_psa_deleted'), table_name='c_psa') + op.create_index('ix_c_preference_show_attributes_uid', 'c_psa', ['uid'], unique=False) + op.create_index('ix_c_preference_show_attributes_deleted', 'c_psa', ['deleted'], unique=False) + op.drop_index(op.f('ix_c_prv_uid'), table_name='c_prv') + op.drop_index(op.f('ix_c_prv_name'), table_name='c_prv') + op.drop_index(op.f('ix_c_prv_deleted'), table_name='c_prv') + op.create_index('ix_c_preference_relation_views_name', 'c_prv', ['name'], unique=False) + op.create_index('ix_c_preference_relation_views_deleted', 'c_prv', ['deleted'], unique=False) + op.alter_column('c_prv', 'uid', + existing_type=mysql.INTEGER(), + nullable=True) + op.drop_index(op.f('ix_c_ci_types_uid'), table_name='c_ci_types') + op.create_index('c_ci_types_uid', 'c_ci_types', ['uid'], unique=False) + op.drop_index(op.f('ix_c_c_t_u_c_deleted'), table_name='c_c_t_u_c') + op.create_index('ix_c_ci_type_unique_constraints_deleted', 'c_c_t_u_c', ['deleted'], unique=False) + op.drop_index(op.f('ix_c_c_t_t_deleted'), table_name='c_c_t_t') + op.create_index('ix_c_ci_type_triggers_deleted', 'c_c_t_t', ['deleted'], unique=False) + op.drop_index(op.f('ix_c_c_d_deleted'), table_name='c_c_d') + op.create_index('ix_c_custom_dashboard_deleted', 'c_c_d', ['deleted'], unique=False) + op.drop_index(op.f('ix_c_attributes_uid'), table_name='c_attributes') + op.create_index('idx_c_attributes_uid', 'c_attributes', ['uid'], unique=False) + op.drop_column('c_attributes', 'choice_other') + op.drop_index(op.f('ix_common_notice_config_deleted'), table_name='common_notice_config') + op.drop_table('common_notice_config') + op.drop_index(op.f('ix_common_data_deleted'), table_name='common_data') + op.drop_table('common_data') + # ### end Alembic commands ### diff --git a/cmdb-api/requirements.txt b/cmdb-api/requirements.txt index 39b32cbe..1776a059 100644 --- a/cmdb-api/requirements.txt +++ b/cmdb-api/requirements.txt @@ -1,80 +1,53 @@ -i https://mirrors.aliyun.com/pypi/simple alembic==1.7.7 -amqp==2.6.1 -aniso8601==9.0.1 -APScheduler==3.10.1 -attrs==23.1.0 -backports.zoneinfo==0.2.1 -bcrypt==4.0.1 -beautifulsoup4==4.12.2 -billiard==3.6.4.0 bs4==0.0.1 -cachelib==0.9.0 -celery==4.3.0 +celery>=5.3.1 celery-once==3.0.1 -certifi==2023.5.7 -charset-normalizer==3.1.0 click==8.1.3 -dnspython==2.3.0 elasticsearch==7.17.9 email-validator==1.3.1 environs==4.2.0 flasgger==0.9.5 -Flask==1.0.3 -Flask-APScheduler==1.12.4 -Flask-Bcrypt==0.7.1 +Flask==2.3.2 +Flask-Bcrypt==1.0.1 Flask-Caching==2.0.2 Flask-Cors==4.0.0 -Flask-Login==0.4.1 +Flask-Login>=0.6.2 Flask-Migrate==2.5.2 -Flask-RESTful==0.3.7 -Flask-SQLAlchemy==2.4.0 -future==0.18.2 -gunicorn==19.5.0 -idna==3.4 -importlib-metadata==6.8.0 -importlib-resources==6.0.0 -itsdangerous==2.0.1 -Jinja2==3.0.1 +Flask-RESTful==0.3.10 +Flask-SQLAlchemy==2.5.0 +future==0.18.3 +gunicorn==21.0.1 +hvac==2.0.0 +itsdangerous==2.1.2 +Jinja2==3.1.2 jinja2schema==0.1.4 jsonschema==4.18.0 -jsonschema-specifications==2023.6.1 -kombu==4.4.0 +kombu>=5.3.1 Mako==1.2.4 MarkupSafe==2.1.3 marshmallow==2.20.2 -meld3==2.0.1 -mistune==3.0.1 more-itertools==5.0.0 msgpack-python==0.5.6 -numpy==1.18.5 -pandas==1.3.2 -Pillow==8.3.2 -pkgutil_resolve_name==1.3.10 -pyasn1==0.5.0 -pyasn1-modules==0.3.0 -pycryptodome==3.12.0 +Pillow>=10.0.1 +cryptography>=41.0.2 PyJWT==2.4.0 -PyMySQL==0.9.3 -python-dateutil==2.8.2 -python-dotenv==1.0.0 -python-ldap==3.2.0 -pytz==2023.3 +PyMySQL==1.1.0 +ldap3==2.9.1 PyYAML==6.0 -redis==3.2.1 -referencing==0.29.1 +redis==4.6.0 requests==2.31.0 -rpds-py==0.8.8 -six==1.12.0 -soupsieve==2.4.1 -SQLAlchemy==1.3.5 +requests_oauthlib==1.3.1 +markdownify==0.11.6 +six==1.16.0 +SQLAlchemy==1.4.49 supervisor==4.0.3 timeout-decorator==0.5.0 toposort==1.10 treelib==1.6.1 -tzlocal==5.0.1 -urllib3==1.26.16 -vine==1.3.0 -Werkzeug==0.15.5 +Werkzeug>=2.3.6 WTForms==3.0.0 -zipp==3.16.0 \ No newline at end of file +shamir~=17.12.0 +hvac~=2.0.0 +pycryptodomex>=3.19.0 +colorama>=0.4.6 diff --git a/cmdb-api/settings.example.py b/cmdb-api/settings.example.py index 3f735170..373e6b78 100644 --- a/cmdb-api/settings.example.py +++ b/cmdb-api/settings.example.py @@ -35,6 +35,7 @@ CACHE_TYPE = "redis" CACHE_REDIS_HOST = "127.0.0.1" CACHE_REDIS_PORT = 6379 +CACHE_REDIS_PASSWORD = "" CACHE_KEY_PREFIX = "CMDB::" CACHE_DEFAULT_TIMEOUT = 3000 @@ -53,13 +54,16 @@ DEFAULT_MAIL_SENDER = '' # # queue -CELERY_RESULT_BACKEND = "redis://127.0.0.1:6379/2" -BROKER_URL = 'redis://127.0.0.1:6379/2' -BROKER_VHOST = '/' +CELERY = { + "broker_url": 'redis://127.0.0.1:6379/2', + "result_backend": "redis://127.0.0.1:6379/2", + "broker_vhost": "/", + "broker_connection_retry_on_startup": True +} ONCE = { 'backend': 'celery_once.backends.Redis', 'settings': { - 'url': BROKER_URL, + 'url': CELERY['broker_url'], } } @@ -76,13 +80,14 @@ AUTH_WITH_LDAP = False LDAP_SERVER = '' LDAP_DOMAIN = '' +LDAP_USER_DN = 'cn={},ou=users,dc=xxx,dc=com' # # pagination DEFAULT_PAGE_COUNT = 50 # # permission WHITE_LIST = ["127.0.0.1"] -USE_ACL = False +USE_ACL = True # # elastic search ES_HOST = '127.0.0.1' @@ -90,4 +95,11 @@ BOOL_TRUE = ['true', 'TRUE', 'True', True, '1', 1, "Yes", "YES", "yes", 'Y', 'y'] -CMDB_API = "http://127.0.0.1:5000/api/v0.1" +# # messenger +USE_MESSENGER = True + +# # secrets +SECRETS_ENGINE = 'inner' # 'inner' or 'vault' +VAULT_URL = '' +VAULT_TOKEN = '' +INNER_TRIGGER_TOKEN = '' diff --git a/cmdb-api/tests/sample.py b/cmdb-api/tests/sample.py index 4e7548be..557ccd70 100644 --- a/cmdb-api/tests/sample.py +++ b/cmdb-api/tests/sample.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- """provide some sample data in database""" -import uuid import random +import uuid +from api.lib.cmdb.ci import CIManager, CIRelationManager +from api.lib.cmdb.ci_type import CITypeAttributeManager +from api.models.acl import User from api.models.cmdb import ( Attribute, CIType, @@ -11,16 +14,12 @@ CITypeRelation, RelationType ) -from api.models.acl import User - -from api.lib.cmdb.ci_type import CITypeAttributeManager -from api.lib.cmdb.ci import CIManager, CIRelationManager def force_add_user(): - from flask import g - if not getattr(g, "user", None): - g.user = User.query.first() + from flask_login import current_user, login_user + if not getattr(current_user, "username", None): + login_user(User.query.first()) def init_attributes(num=1): @@ -77,12 +76,12 @@ def init_relation_type(num=1): def init_ci_type_relation(num=1): result = [] - ci_types = init_ci_types(num+1) + ci_types = init_ci_types(num + 1) relation_types = init_relation_type(num) for i in range(num): result.append(CITypeRelation.create( parent_id=ci_types[i].id, - child_id=ci_types[i+1].id, + child_id=ci_types[i + 1].id, relation_type_id=relation_types[i].id )) return result diff --git a/cmdb-ui/.env b/cmdb-ui/.env index ee9f4c75..1be71b79 100644 --- a/cmdb-ui/.env +++ b/cmdb-ui/.env @@ -1,5 +1,5 @@ NODE_ENV=production VUE_APP_PREVIEW=false -VUE_APP_API_BASE_URL=/api +VUE_APP_API_BASE_URL=http://127.0.0.1:5000/api VUE_APP_BUILD_PACKAGES="ticket,calendar,acl" VUE_APP_IS_OUTER=true diff --git a/cmdb-ui/.gitattributes b/cmdb-ui/.gitattributes deleted file mode 100644 index e5073192..00000000 --- a/cmdb-ui/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -public/* linguist-vendored \ No newline at end of file diff --git a/cmdb-ui/.gitignore b/cmdb-ui/.gitignore deleted file mode 100644 index 666e7499..00000000 --- a/cmdb-ui/.gitignore +++ /dev/null @@ -1,26 +0,0 @@ -.DS_Store -node_modules -/dist -/dist.zip -/temp - -# local env files -.env.local -.env.*.local - -# Log files -npm-debug.log* -yarn-debug.log* -yarn-error.log* - -# Editor directories and files -.idea -.vscode -*.suo -*.ntvs* -*.njsproj -*.sln -*.sw* -*.css.map - -.env.development \ No newline at end of file diff --git a/cmdb-ui/README.zh-CN.md b/cmdb-ui/README.zh-CN.md deleted file mode 100644 index 835f5530..00000000 --- a/cmdb-ui/README.zh-CN.md +++ /dev/null @@ -1,11 +0,0 @@ -#Oneops-UI - -```shell -## build -yarn run build - -## develop -yarn run serve - - -``` \ No newline at end of file diff --git a/cmdb-ui/package.json b/cmdb-ui/package.json index f3c6a510..6aec35de 100644 --- a/cmdb-ui/package.json +++ b/cmdb-ui/package.json @@ -17,8 +17,10 @@ "@babel/plugin-syntax-import-meta": "^7.10.4", "@riophae/vue-treeselect": "^0.4.0", "@vue/composition-api": "^1.7.1", + "@wangeditor/editor": "^5.1.23", + "@wangeditor/editor-for-vue": "^1.0.0", "ant-design-vue": "^1.6.5", - "axios": "0.18.0", + "axios": "1.6.0", "babel-eslint": "^8.2.2", "butterfly-dag": "^4.3.26", "codemirror": "^5.65.13", @@ -37,6 +39,7 @@ "moment": "^2.24.0", "nprogress": "^0.2.0", "relation-graph": "^1.1.0", + "snabbdom": "^3.5.1", "sortablejs": "1.9.0", "viser-vue": "^2.4.8", "vue": "2.6.11", @@ -53,21 +56,22 @@ "vuedraggable": "^2.23.0", "vuex": "^3.1.1", "vxe-table": "3.6.9", - "vxe-table-plugin-export-xlsx": "^3.0.4", + "vxe-table-plugin-export-xlsx": "2.0.0", "xe-utils": "3", "xlsx": "0.15.0", "xlsx-js-style": "^1.2.0" }, "devDependencies": { "@ant-design/colors": "^3.2.2", + "@babel/core": "^7.23.2", "@babel/polyfill": "^7.2.5", + "@babel/preset-env": "^7.23.2", "@vue/cli-plugin-babel": "4.5.17", "@vue/cli-plugin-eslint": "^4.0.5", "@vue/cli-plugin-unit-jest": "^4.0.5", "@vue/cli-service": "^4.0.5", "@vue/eslint-config-standard": "^4.0.0", "@vue/test-utils": "^1.0.0-beta.30", - "babel-core": "7.0.0-bridge.0", "babel-jest": "^23.6.0", "babel-plugin-import": "^1.11.0", "babel-plugin-transform-remove-console": "^6.9.4", diff --git a/cmdb-ui/public/iconfont/demo_index.html b/cmdb-ui/public/iconfont/demo_index.html index f7d3d470..ccdb1b84 100644 --- a/cmdb-ui/public/iconfont/demo_index.html +++ b/cmdb-ui/public/iconfont/demo_index.html @@ -54,6 +54,168 @@