Feat a76yyyy#29 新建任务时自动提取模板变量 default 值

.gitignore

@@ -182,4 +182,6 @@ node_modules
 pnpm-global
 TODOs.md
 
-cache
+cache
+
+alembic.ini

Pipfile

@@ -19,12 +19,12 @@ pysocks = "*"
 faker = "*"
 colorama = {version = "*", markers="sys_platform == 'win32'"}
 aiofiles = "*"
-sqlalchemy = {version = "*", extras = ["asyncio"]}
 passlib = "*"
 aiomysql = "*"
 aiosqlite = "*"
 cryptography = "*"
 aiohttp = "*"
+sqlalchemy = {version = "<2.0", extras = ["asyncio"]}
 
 [dev-packages]
 opencv-python-headless = "*"

backup.py

@@ -135,7 +135,7 @@ def new(self, userid, maindb):
                                       'mip', 'aip', 'skey', 'barkurl', 'wxpusher', 'noticeflg', 'logtime', 'status', 'notepad', 'diypusher', 'qywx_token', 'tg_token', 'dingding_token', 'qywx_webhook', 'push_batch'))
             userkey = maindb.db.user.__getuserkey(user['env'])
             tpls = []
-            for tpl in maindb.db.tpl.list(fields=('id', 'userid', 'siteurl', 'sitename', 'banner', 'disabled', 'public', 'lock', 'fork', 'har', 'tpl', 'variables', 'interval', 'note', 'success_count', 'failed_count', 'last_success', 'ctime', 'mtime', 'atime', 'tplurl', 'updateable', '_groups'), limit=None):
+            for tpl in maindb.db.tpl.list(fields=('id', 'userid', 'siteurl', 'sitename', 'banner', 'disabled', 'public', 'lock', 'fork', 'har', 'tpl', 'variables', 'interval', 'note', 'success_count', 'failed_count', 'last_success', 'ctime', 'mtime', 'atime', 'tplurl', 'updateable', '_groups', 'init_env'), limit=None):
                 if tpl['userid'] == userid:
                     tpls.append(tpl)
             tasks = []

db/basedb.py

@@ -28,7 +28,8 @@
     user=config.mysql.user
     passwd=config.mysql.passwd
     auth_plugin=config.mysql.auth_plugin
-    engine = create_async_engine(f"mysql+aiomysql://{user}:{passwd}@{host}:{port}/{database}?auth_plugin={auth_plugin}",
+    engine_url = f"mysql+aiomysql://{user}:{passwd}@{host}:{port}/{database}?auth_plugin={auth_plugin}"
+    engine = create_async_engine(engine_url,
                                 logging_name = config.sqlalchemy.logging_name,
                                 pool_size = config.sqlalchemy.pool_size,
                                 max_overflow = config.sqlalchemy.max_overflow,
@@ -38,7 +39,8 @@
                                 pool_timeout = config.sqlalchemy.pool_timeout,
                                 pool_use_lifo = config.sqlalchemy.pool_use_lifo)
 elif config.db_type == 'sqlite3':
-    engine = create_async_engine(f"sqlite+aiosqlite:///{config.sqlite3.path}", 
+    engine_url = f"sqlite+aiosqlite:///{config.sqlite3.path}"
+    engine = create_async_engine(engine_url, 
                                 logging_name = config.sqlalchemy.logging_name,
                                 pool_logging_name = config.sqlalchemy.pool_logging_name,
                                 pool_pre_ping = config.sqlalchemy.pool_pre_ping,

db/db_converter.py

@@ -12,7 +12,6 @@
 
 import config
 from db import DB, Site, Task, Tpl, User
-from db.basedb import BaseDB
 from libs import mcrypto as crypto
 from libs.log import Log
 
@@ -84,7 +83,8 @@ async def ConvertNewType(self, db=DB(), path=config.sqlite3.path):
             `atime` INT UNSIGNED NOT NULL,
             `tplurl` VARCHAR(1024) NULL DEFAULT '',
             `updateable` INT UNSIGNED NOT NULL DEFAULT 0,
-            `_groups` VARCHAR(256) NOT NULL DEFAULT 'None'
+            `_groups` VARCHAR(256) NOT NULL DEFAULT 'None',
+            `init_env` TEXT NULL
             );'''% autokey)
             await exec_shell('''CREATE TABLE IF NOT EXISTS `task` (
             `id` INTEGER NOT NULL PRIMARY KEY %s,
@@ -669,5 +669,37 @@ async def ConvertNewType(self, db=DB(), path=config.sqlite3.path):
             result = await self.db._update(update(Tpl).where(Tpl.userid == None).where(Tpl.public == 0).values(public=1))
         except Exception as e:
             logger_DB_converter.debug(e)
+
+        try:
+            await self.db.tpl.list(limit=1, fields=('init_env',))
+        except Exception as e:
+            logger_DB_converter.debug(e)
+            await exec_shell("ALTER TABLE `tpl` ADD `init_env` TEXT NULL" )
+            from jinja2.nodes import Filter, Name
+
+            from libs.fetcher import Fetcher
+            env = Fetcher().jinja_env
+            async with self.db.transaction() as sql_session:
+                tpls = await self.db.tpl.list(fields=('id', 'userid', 'tpl', 'variables', 'init_env'), sql_session=sql_session)
+                for tpl in tpls:
+                    if not tpl['init_env']:
+                        if not tpl['variables']:
+                            tpl['variables'] = '[]'
+                        variables = json.loads(tpl['variables'])
+                        init_env = {}
+                        if variables:
+                            _tpl = await self.db.user.decrypt(0 if not tpl['userid'] else tpl['userid'], tpl['tpl'], sql_session=sql_session)
+                            try:
+                                ast = env.parse(_tpl)
+                                for x in ast.find_all(Filter):
+                                    if x.name == 'default' and isinstance(x.node, Name) and len(x.args) > 0 and x.node.name in variables and x.node.name not in init_env:
+                                        try:
+                                            init_env[x.node.name] = x.args[0].as_const()
+                                        except Exception as e:
+                                            logger_DB_converter.debug('Convert init_env error: %s' % e)
+                            except Exception as e:
+                                logger_DB_converter.debug('Convert init_env error: %s' % e)
+                        await self.db.tpl.mod(tpl['id'], init_env=json.dumps(init_env), sql_session=sql_session)
+
 
         return 

db/tpl.py

@@ -42,12 +42,13 @@ class Tpl(BaseDB,AlchemyMixin):
     har = Column(MEDIUMBLOB)
     tpl = Column(MEDIUMBLOB)
     variables = Column(Text)
+    init_env = Column(Text)
     interval = Column(INTEGER)
     note = Column(String(1024))
     last_success = Column(INTEGER)
     tplurl = Column(String(1024), server_default=text("''"))
 
-    def add(self, userid, har, tpl, variables, interval=None, sql_session=None):
+    def add(self, userid, har, tpl, variables, init_env, interval=None, sql_session=None):
         now = time.time()
 
         insert = dict(
@@ -61,6 +62,7 @@ def add(self, userid, har, tpl, variables, interval=None, sql_session=None):
                 har = har,
                 tpl = tpl,
                 variables = variables,
+                init_env = init_env,
                 interval = interval,
                 ctime = now,
                 mtime = now,

requirements.txt

@@ -6,43 +6,42 @@
 #
 
 -i https://pypi.tuna.tsinghua.edu.cn/simple
-aiofiles==22.1.0
-aiohttp==3.8.3
+aiofiles==23.1.0
+aiohttp==3.8.4
 aiomysql==0.1.1
 aiosignal==1.3.1; python_version >= '3.7'
 aiosqlite==0.18.0
 async-timeout==4.0.2; python_version >= '3.6'
 attrs==22.2.0; python_version >= '3.6'
 certifi==2022.12.7; python_version >= '3.6'
 cffi==1.15.1
-charset-normalizer==2.1.1
+charset-normalizer==3.0.1
 colorama==0.4.6; sys_platform == 'win32'
 croniter==1.3.8
-cryptography==39.0.0
-faker==16.1.0
+cryptography==39.0.1
+faker==17.0.0
 frozenlist==1.3.3; python_version >= '3.7'
-greenlet==2.0.1; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
+greenlet==2.0.2; python_version >= '3' and platform_machine == 'aarch64' or (platform_machine == 'ppc64le' or (platform_machine == 'x86_64' or (platform_machine == 'amd64' or (platform_machine == 'AMD64' or (platform_machine == 'win32' or platform_machine == 'WIN32')))))
 idna==3.4; python_version >= '3.5'
 incremental==22.10.0
 jinja2==3.1.2
-markupsafe==2.1.1; python_version >= '3.7'
+markupsafe==2.1.2; python_version >= '3.7'
 multidict==6.0.4; python_version >= '3.7'
 passlib==1.7.4
 pbkdf2==1.3
 pycparser==2.21
-pycryptodome==3.16.0
+pycryptodome==3.17
 pymysql==1.0.2; python_version >= '3.6'
 pysocks==1.7.1
 python-dateutil==2.8.2; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
-pytz==2022.7
-redis==4.4.1
-requests==2.28.1
+pytz==2022.7.1
+redis==4.5.1
+requests==2.28.2
 six==1.16.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
 sqlalchemy[asyncio]==1.4.46
 tornado==6.2
 u-msgpack-python==2.7.2
-urllib3==1.26.13; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'
-yarl==1.8.2; python_version >= '3.7'
+urllib3==1.26.14; python_version >= 
 # ddddocr==1.4.7
 # opencv-python-headless==4.7.0.68
 # pycurl==7.45.2

web/handlers/base.py

@@ -22,6 +22,7 @@
 
 class BaseHandler(tornado.web.RequestHandler):
     application_export = set(('db', 'fetcher'))
+    db:DB
     # db = DB()
     def __getattr__(self, key):
         if key in self.application_export:

web/handlers/har.py

@@ -14,6 +14,7 @@
 
 import umsgpack
 from jinja2 import Environment, meta
+from jinja2.nodes import Const, Filter, Getattr, List, Name, Tuple
 from tornado import gen, httpclient
 
 from libs import utils
@@ -48,18 +49,18 @@ async def post(self, id):
 
         async with self.db.transaction() as sql_session:
             tpl = self.check_permission(
-                    await self.db.tpl.get(id, fields=('id', 'userid', 'sitename', 'siteurl', 'banner', 'note', 'interval', 'har', 'variables', 'lock'), sql_session=sql_session))
+                    await self.db.tpl.get(id, fields=('id', 'userid', 'sitename', 'siteurl', 'banner', 'note', 'interval', 'har', 'variables', 'lock', 'init_env'), sql_session=sql_session))
 
             tpl['har'] = await self.db.user.decrypt(tpl['userid'], tpl['har'], sql_session=sql_session)
             tpl['variables'] = json.loads(tpl['variables'])
+            if not tpl['init_env']:
+                tpl['init_env'] = '{}'
+            envs = json.loads(tpl['init_env'])
             if taskid:
                 task = await self.db.task.get(taskid, sql_session=sql_session)
                 if task['init_env']:
-                    envs = await self.db.user.decrypt(user['id'], task['init_env'], sql_session=sql_session)
-                else:
-                    envs = {}
-            else:
-                envs = {}
+                    task_envs = await self.db.user.decrypt(user['id'], task['init_env'], sql_session=sql_session)
+                    envs.update(task_envs)
 
             #await self.db.tpl.mod(id, atime=time.time(), sql_session=sql_session)
         await self.finish(dict(
@@ -120,11 +121,11 @@ async def post(self):
         await self.finish(result)
 
 class HARSave(BaseHandler):
+    env = Fetcher().jinja_env
     @staticmethod
-    def get_variables(tpl):
+    def get_variables(env, tpl):
         variables = set()
         extracted = set(utils.jinja_globals.keys())
-        env = Fetcher().jinja_env
         for entry in tpl:
             req = entry['request']
             rule = entry['rule']
@@ -169,7 +170,20 @@ async def post(self, id):
         async with self.db.transaction() as sql_session:
             har = await self.db.user.encrypt(userid, data['har'], sql_session=sql_session)
             tpl = await self.db.user.encrypt(userid, data['tpl'], sql_session=sql_session)
-            variables = json.dumps(list(self.get_variables(data['tpl'])))
+            variables = list(self.get_variables(self.env,data['tpl']))
+            init_env = {}
+            try:
+                ast = self.env.parse(data['tpl'])
+                for x in ast.find_all(Filter):
+                    if x.name == 'default' and isinstance(x.node, Name) and len(x.args) > 0 and x.node.name in variables and x.node.name not in init_env:
+                        try:
+                            init_env[x.node.name] = x.args[0].as_const()
+                        except Exception as e:
+                            logger_Web_Handler.debug('HARSave init_env error: %s' % e)
+            except Exception as e:
+                logger_Web_Handler.debug('HARSave ast error: %s' % e)
+            variables = json.dumps(variables)
+            init_env = json.dumps(init_env)
             groupName = 'None'
             if id:
                 _tmp = self.check_permission(await self.db.tpl.get(id, fields=('id', 'userid', 'lock'), sql_session=sql_session), 'w')
@@ -182,11 +196,11 @@ async def post(self, id):
                     await self.finish(u'模板已锁定')
                     return
 
-                await self.db.tpl.mod(id, har=har, tpl=tpl, variables=variables, sql_session=sql_session)
+                await self.db.tpl.mod(id, har=har, tpl=tpl, variables=variables, init_env=init_env, sql_session=sql_session)
                 groupName = (await self.db.tpl.get(id, fields=('_groups',), sql_session=sql_session))['_groups']
             else:
                 try:
-                    id = await self.db.tpl.add(userid, har, tpl, variables, sql_session=sql_session)
+                    id = await self.db.tpl.add(userid, har, tpl, variables, init_env=init_env, sql_session=sql_session)
                 except Exception as e:
                     if "max_allowed_packet" in str(e):
                         raise Exception('har大小超过MySQL max_allowed_packet 限制; \n'+str(e))

web/handlers/push.py

@@ -120,7 +120,7 @@ async def post(self, prid, action):
         self.redirect('/pushs')
 
     async def accept(self, pr, sql_session=None):
-        tplobj = await self.db.tpl.get(pr['from_tplid'], fields=('id', 'userid', 'tpl', 'variables', 'sitename', 'siteurl', 'note', 'banner', 'interval'), sql_session=sql_session)
+        tplobj = await self.db.tpl.get(pr['from_tplid'], fields=('id', 'userid', 'tpl', 'variables', 'sitename', 'siteurl', 'note', 'banner', 'interval', 'init_env'), sql_session=sql_session)
         if not tplobj:
             self.cancel(pr)
             raise HTTPError(404)
@@ -137,6 +137,7 @@ async def accept(self, pr, sql_session=None):
                     har = har,
                     tpl = tpl,
                     variables = tplobj['variables'],
+                    init_env=tplobj['init_env'],
                     interval = tplobj['interval'],
                     sql_session=sql_session
                     )
@@ -156,6 +157,7 @@ async def accept(self, pr, sql_session=None):
                     tpl = tpl,
                     public = 1,
                     variables = tplobj['variables'],
+                    init_env = tplobj['init_env'],
                     interval = tplobj['interval'],
                     sitename = tplobj['sitename'],
                     siteurl = tplobj['siteurl'],

web/handlers/task.py

@@ -43,8 +43,11 @@ async def get(self):
         if tplid:
             tplid = int(tplid)
 
-            tpl = self.check_permission(await self.db.tpl.get(tplid, fields=('id', 'userid', 'note', 'sitename', 'siteurl', 'variables')))
+            tpl = self.check_permission(await self.db.tpl.get(tplid, fields=('id', 'userid', 'note', 'sitename', 'siteurl', 'variables','init_env')))
             variables = json.loads(tpl['variables'])
+            if not tpl['init_env']:
+                tpl['init_env'] = '{}'
+            init_env = json.loads(tpl['init_env'])
 
             _groups = []
             if user:
@@ -55,7 +58,7 @@ async def get(self):
                     if (temp not  in _groups):
                         _groups.append(temp)
 
-            await self.render('task_new.html', tpls=tpls, tplid=tplid, tpl=tpl, variables=variables, task={}, _groups=_groups, init_env=tpl['variables'], default_retry_count=config.task_max_retry_count)
+            await self.render('task_new.html', tpls=tpls, tplid=tplid, tpl=tpl, variables=variables, task={}, _groups=_groups, init_env=init_env, default_retry_count=config.task_max_retry_count)
         else:
             await self.render('utils_run_result.html', log=u'请先添加模板！', title=u'设置失败', flg='danger')
 

web/handlers/tpl.py

@@ -66,12 +66,14 @@ async def post(self, tplid):
 class TPLVarHandler(BaseHandler):
     async def get(self, tplid):
         user = self.current_user
-        tpl = await self.db.tpl.get(tplid, fields=('id', 'note', 'userid', 'sitename', 'siteurl', 'variables'))
+        tpl = await self.db.tpl.get(tplid, fields=('id', 'note', 'userid', 'sitename', 'siteurl', 'variables', 'init_env'))
         if not self.permission(tpl):
             self.evil(+5)
             await self.finish('<span class="alert alert-danger">没有权限</span>')
             return
-        await self.render('task_new_var.html', tpl=tpl, variables=json.loads(tpl['variables']))
+        if not tpl['init_env']:
+            tpl['init_env'] = '{}'
+        await self.render('task_new_var.html', tpl=tpl, variables=json.loads(tpl['variables']), init_env=json.loads(tpl['init_env']))
 
 class TPLDelHandler(BaseHandler):
     @tornado.web.authenticated

web/handlers/user.py

@@ -469,7 +469,7 @@ async def post(self, userid):
                 if await self.db.user.challenge_MD5(mail, pwd, sql_session=sql_session) and (user['email'] == mail):
                     if ('backuptplsbtn' in envs):
                         tpls = []
-                        for tpl in await self.db.tpl.list(userid=userid, fields=('id', 'siteurl', 'sitename', 'banner', 'note','fork', '_groups', 'har', 'tpl', 'variables'), limit=None, sql_session=sql_session):
+                        for tpl in await self.db.tpl.list(userid=userid, fields=('id', 'siteurl', 'sitename', 'banner', 'note','fork', '_groups', 'har', 'tpl', 'variables','init_env'), limit=None, sql_session=sql_session):
                             tpl['tpl'] = await self.db.user.decrypt(userid, tpl['tpl'], sql_session=sql_session)
                             tpl['har'] = await self.db.user.decrypt(userid, tpl['har'], sql_session=sql_session)
                             tpls.append(tpl)
@@ -519,7 +519,8 @@ async def post(self, userid):
                                 har = await self.db.user.encrypt(userid2, newtpl['har'], sql_session=sql_session)
                                 tpl = await self.db.user.encrypt(userid2, newtpl['tpl'], sql_session=sql_session)
                                 variables = newtpl['variables']
-                                newid = await self.db.tpl.add(userid2, har, tpl, variables, sql_session=sql_session)
+                                init_env = newtpl.get('init_env', "{}")
+                                newid = await self.db.tpl.add(userid2, har, tpl, variables, init_env=init_env, sql_session=sql_session)
                                 await self.db.tpl.mod(newid, fork = newtpl['fork'],
                                                     siteurl = newtpl['siteurl'],
                                                     sitename = newtpl['sitename'],

web/tpl/task_new_var.html

@@ -33,7 +33,14 @@
 {% for var in variables %}
 <div class="form-group">
   <label class="control-label" for="input-{{ var }}">{{ varname_format(var) }}</label>
-  <input type="text" class="form-control" name="{{ var }}" id="input-{{ var }}" placeholder="请输入 {{var}}">
+  {% if init_env.get(var, '') %}
+    <input type="text" class="form-control" name="{{ var }}" id="input-{{ var }}" value="{{ init_env.get(var, '') }}">
+    默认值：<div class="autowrap showbut" id="pvar-{{ var }}">{{ init_env.get(var, '') }}
+      <div class="btn hljs-button" data-clipboard-target="#pvar-{{ var }}" style="vertical-align:top">复制</div>
+    </div>
+  {% else %}
+    <input type="text" class="form-control" name="{{ var }}" id="input-{{ var }}" placeholder="请输入 {{var}}">
+  {% endif %}
   <!-- 未登录首页-加图标 -->
   <span class="glyphicon glyphicon-file form-control-feedback" aria-hidden="true"></span>
   {{ var_tips(var) }}