diff --git a/README.rst b/README.rst index ba40f3b..0d1701e 100644 --- a/README.rst +++ b/README.rst @@ -192,6 +192,8 @@ Extensions +==============+==============================================+ | len | - $.objects.`len` | +--------------+----------------------------------------------+ +| keys | - $.dict_field.`keys`(returns a list of keys)| ++--------------+----------------------------------------------+ | sub | - $.field.`sub(/foo\\\\+(.*)/, \\\\1)` | +--------------+----------------------------------------------+ | split | - $.field.`split(+, 2, -1)` | @@ -203,7 +205,10 @@ Extensions +--------------+----------------------------------------------+ | filter | - $.objects[?(@some_field > 5)] | | | - $.objects[?some_field = "foobar")] | -| | - $.objects[?some_field > 5 & other < 2)] | +| | - $.objects[?some_field > 5 & other < 2)] and| +| | - $.objects[?some_field>5 |some_field<2)] or | +| | - $.objects[?(!(field>5 | field<2))] not| +| | - $.objects[?@.field ~= "a.+a"] regex| +--------------+----------------------------------------------+ | arithmetic | - $.foo + "_" + $.bar | | (-+*/) | - $.foo * 12 | diff --git a/jsonpath_ng/ext/filter.py b/jsonpath_ng/ext/filter.py index 4be473c..7a1ef64 100644 --- a/jsonpath_ng/ext/filter.py +++ b/jsonpath_ng/ext/filter.py @@ -13,9 +13,15 @@ import operator from six import moves +import re from .. import JSONPath, DatumInContext, Index +def contains(a,b): + if re.search(b,a): + return True + return False + OPERATOR_MAP = { '!=': operator.ne, @@ -25,8 +31,37 @@ '<': operator.lt, '>=': operator.ge, '>': operator.gt, + '~=': contains } +def eval_exp(expressions,val): + for expression in expressions: + if type(expression)==tuple and expression[0]=='|': + val1=eval_exp(expression[1],val) + val2=eval_exp(expression[2],val) + if (val1 or val2): + return True + else: + return False + if type(expression)==tuple and expression[0]=='&': + val1=eval_exp(expression[1],val) + val2=eval_exp(expression[2],val) + if (val1 and val2): + return True + else: + return False + if type(expression)==tuple and expression[0]=='!': + val1=eval_exp(expression[1],val) + if (val1): + return False + else: + return True + else: + if(len([expression])==len(list(filter(lambda x: x.find(val),[expression])))): + return True + else: + return False + class Filter(JSONPath): """The JSONQuery filter""" @@ -41,12 +76,11 @@ def find(self, datum): datum = DatumInContext.wrap(datum) if not isinstance(datum.value, list): return [] - - return [DatumInContext(datum.value[i], path=Index(i), context=datum) - for i in moves.range(0, len(datum.value)) - if (len(self.expressions) == - len(list(filter(lambda x: x.find(datum.value[i]), - self.expressions))))] + res=[] + for i in moves.range(0,len(datum.value)): + if eval_exp(self.expressions,datum.value[i]): + res.append(DatumInContext(datum.value[i], path=Index(i), context=datum)) + return(res) def __repr__(self): return '%s(%r)' % (self.__class__.__name__, self.expressions) @@ -65,7 +99,6 @@ def __init__(self, target, op, value): def find(self, datum): datum = self.target.find(DatumInContext.wrap(datum)) - if not datum: return [] if self.op is None: diff --git a/jsonpath_ng/ext/iterable.py b/jsonpath_ng/ext/iterable.py index 92ece5f..e5ddbfc 100644 --- a/jsonpath_ng/ext/iterable.py +++ b/jsonpath_ng/ext/iterable.py @@ -90,3 +90,29 @@ def __str__(self): def __repr__(self): return 'Len()' + +class Keys(JSONPath): + """The JSONPath referring to the keys of the current object. + + Concrete syntax is '`keys`'. + """ + + def find(self, datum): + datum = DatumInContext.wrap(datum) + try: + value = datum.value.keys() + except Exception as e: + return [] + else: + return [DatumInContext(value[i], + context=None, + path=Keys()) for i in range (0, len(datum.value))] + + def __eq__(self, other): + return isinstance(other, Keys) + + def __str__(self): + return '`keys`' + + def __repr__(self): + return 'Keys()' diff --git a/jsonpath_ng/ext/parser.py b/jsonpath_ng/ext/parser.py index c7d7cae..9da08a7 100644 --- a/jsonpath_ng/ext/parser.py +++ b/jsonpath_ng/ext/parser.py @@ -14,7 +14,6 @@ from .. import lexer from .. import parser from .. import Fields, This, Child - from . import arithmetic as _arithmetic from . import filter as _filter from . import iterable as _iterable @@ -23,12 +22,11 @@ class ExtendedJsonPathLexer(lexer.JsonPathLexer): """Custom LALR-lexer for JsonPath""" - literals = lexer.JsonPathLexer.literals + ['?', '@', '+', '*', '/', '-'] + literals = lexer.JsonPathLexer.literals + ['?', '@', '+', '*', '/', '-', '!','~'] tokens = (['BOOL'] + parser.JsonPathLexer.tokens + ['FILTER_OP', 'SORT_DIRECTION', 'FLOAT']) - - t_FILTER_OP = r'==?|<=|>=|!=|<|>' + t_FILTER_OP = r'==?|<=|>=|!=|<|>|~=' def t_BOOL(self, t): r'true|false' @@ -94,6 +92,8 @@ def p_jsonpath_named_operator(self, p): "jsonpath : NAMED_OPERATOR" if p[1] == 'len': p[0] = _iterable.Len() + elif p[1] == 'keys': + p[0] = _iterable.Keys() elif p[1] == 'sorted': p[0] = _iterable.SortedThis() elif p[1].startswith("split("): @@ -119,11 +119,18 @@ def p_expression(self, p): def p_expressions_expression(self, p): "expressions : expression" p[0] = [p[1]] - + + def p_expressions_not(self, p): + "expressions : '!' expressions" + p[0]=[('!',p[2])] + def p_expressions_and(self, p): "expressions : expressions '&' expressions" - # TODO(sileht): implements '|' - p[0] = p[1] + p[3] + p[0] = [('&',p[1],p[3])] + + def p_expressions_or(self, p): + "expressions : expressions '|' expressions" + p[0] = [('|',p[1],p[3])] def p_expressions_parens(self, p): "expressions : '(' expressions ')'" diff --git a/tests/test_jsonpath_rw_ext.py b/tests/test_jsonpath_rw_ext.py index fb3c886..192e5c6 100644 --- a/tests/test_jsonpath_rw_ext.py +++ b/tests/test_jsonpath_rw_ext.py @@ -25,7 +25,53 @@ import testscenarios from jsonpath_ng.ext import parser - +rest_response1={ + "items": + [ + { + "status": "UP", + "kind": "compute#region", + "description": "us-central1", + "quotas": [ + {"usage": 3.0, "metric": "CPUS", "limit": 72.0}, + {"usage": 261.0, "metric": "DISKS", "limit": 40960.0}, + {"usage": 0.0, "metric": "STATIC", "limit": 21.0}, + {"usage": 0.0, "metric": "IN_USE", "limit": 69.0}, + {"usage": 0.0, "metric": "SSD", "limit": 20480.0} + ], + "id": "1000", + "name": "us-central1" + }, + { + "status": "UP", + "kind": "compute#region", + "description": "us-central2", + "quotas": [ + {"usage": 0.0, "metric": "CPUS", "limit": 72.0}, + {"usage": 0.0, "metric": "DISKS", "limit": 40960.0}, + {"usage": 0.0, "metric": "STATIC", "limit": 21.0}, + {"usage": 0.0, "metric": "IN_USE", "limit": 69.0}, + {"usage": 0.0, "metric": "SSD", "limit": 20480.0} + ], + "id": "1001", + "name": "us-central2" + }, + { + "status": "UP", + "kind": "compute#region", + "description": "us-central3", + "quotas": [ + {"usage": 0.0, "metric": "CPUS", "limit": 90.0}, + {"usage": 0.0, "metric": "DISKS", "limit": 2040.0}, + {"usage": 0.0, "metric": "STATIC", "limit": 46.0}, + {"usage": 0.0, "metric": "IN_USE", "limit": 80.0}, + {"usage": 500.0, "metric": "SSD", "limit": 20480.0} + ], + "id": "1002", + "name": "us-central3" + } + ] +} class Testjsonpath_ng_ext(testscenarios.WithScenarios, base.BaseTestCase): @@ -50,12 +96,31 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios, ('len_list', dict(string='objects.`len`', data={'objects': ['alpha', 'gamma', 'beta']}, target=3)), + ('filter_list', dict(string='objects[?@="alpha"]', + data={'objects': ['alpha', 'gamma', 'beta']}, + target=['alpha'])), + ('filter_list_2', dict(string='objects[?@ ~= "a.+"]', + data={'objects': ['alpha', 'gamma', 'beta']}, + target=['alpha','gamma'])), + ('keys_list', dict(string='objects.`keys`', + data={'objects': ['alpha', 'gamma', 'beta']}, + target=[])), ('len_dict', dict(string='objects.`len`', data={'objects': {'cow': 'moo', 'cat': 'neigh'}}, target=2)), + ('keys_dict', dict(string='objects.`keys`', + data={'objects': {'cow': 'moo', 'cat': 'neigh'}}, + target=['cow','cat'])), + #('filter_keys_dict', dict(string='objects.`keys`[?`this`="cow"]', + # data={'objects': {'cow': 'moo', 'cat': 'neigh'}}, + # target=['cow'])), + #TODO make keys dictionaries filterable ('len_str', dict(string='objects[0].`len`', data={'objects': ['alpha', 'gamma']}, target=5)), + ('contains_filter', dict(string='objects[?id ~= "v.*[1-9]"].id', + data={'objects': [{'id':'vasll1'},{'id':'v2'},{'id':'vaal3'},{'id':'other'},{'id':'val'}]}, + target=['vasll1','v2','vaal3'])), ('filter_exists_syntax1', dict(string='objects[?cow]', data={'objects': [{'cow': 'moo'}, @@ -94,6 +159,12 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios, {'cow': 5}, {'cow': 'neigh'}]}, target=[{'cow': 8}, {'cow': 7}])), + ('filter_gt_negation', dict(string='objects[?!cow<=5]', + data={'objects': [{'cow': 8}, + {'cow': 7}, + {'cow': 5}, + {'cow': 'neigh'}]}, + target=[{'cow': 8}, {'cow': 7},{'cow':'neigh'}])), ('filter_and', dict(string='objects[?cow>5&cat=2]', data={'objects': [{'cow': 8, 'cat': 2}, {'cow': 7, 'cat': 2}, @@ -102,6 +173,85 @@ class Testjsonpath_ng_ext(testscenarios.WithScenarios, {'cow': 8, 'cat': 3}]}, target=[{'cow': 8, 'cat': 2}, {'cow': 7, 'cat': 2}])), + ('filter_and_demorgans', dict(string='objects[?!(cow<=5|cat!=2)]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}])), + ('filter_or', dict(string='objects[?cow=8|cat=3]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}])), + ('filter_or_demorgans', dict(string='objects[?!(cow!=8&cat!=3)]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}])), + ('filter_or_and', dict(string='objects[?cow=8&cat=2|cat=3]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}])), + ('filter_or_and_overide', dict(string='objects[?cow=8&(cat=2|cat=3)]', + data={'objects': [{'cow': 8, 'cat': 2}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2}, + {'cow': 8, 'cat': 3}])), + ('filter_or_and', dict(string='objects[?dog=1|cat=3&cow=8]', + data={'objects': [{'cow': 8, 'cat': 2, 'dog':1}, + {'cow': 7, 'cat': 2}, + {'cow': 2, 'cat': 2}, + {'cow': 5, 'cat': 3}, + {'cow': 8, 'cat': 3}]}, + target=[{'cow': 8, 'cat': 2, 'dog':1}, + {'cow': 8, 'cat': 3}])), + ('filter_complex', dict(string='$.items[?((!(val==4))&(id==2))|(!((id!=1)&(id!=3)))]', + data={"items":[{"id":1, "val":1, "info":1},{"id":2, "val":4},{"id":2,"val":2},{"id":3,"val":3}]}, + target=[{'info': 1, 'id': 1, 'val': 1}, + {'id': 2, 'val': 2}, + {'id': 3, 'val': 3}])), + ('filter_complex2', dict(string="$.items[?(@.quotas[?((@.metric='SSD' & @.usage>0) | (@.metric='CPU' & @.usage>0) | (@.metric='DISKS' & @.usage>0))])]", + data=rest_response1, + target=[{'description': 'us-central1', + 'id': '1000', + 'kind': 'compute#region', + 'name': 'us-central1', + 'quotas': [{'limit': 72.0, 'metric': 'CPUS', 'usage': 3.0}, + {'limit': 40960.0, 'metric': 'DISKS', 'usage': 261.0}, + {'limit': 21.0, 'metric': 'STATIC', 'usage': 0.0}, + {'limit': 69.0, 'metric': 'IN_USE', 'usage': 0.0}, + {'limit': 20480.0, 'metric': 'SSD', 'usage': 0.0}], + 'status': 'UP'}, + {'description': 'us-central3', + 'id': '1002', + 'kind': 'compute#region', + 'name': 'us-central3', + 'quotas': [{'limit': 90.0, 'metric': 'CPUS', 'usage': 0.0}, + {'limit': 2040.0, 'metric': 'DISKS', 'usage': 0.0}, + {'limit': 46.0, 'metric': 'STATIC', 'usage': 0.0}, + {'limit': 80.0, 'metric': 'IN_USE', 'usage': 0.0}, + {'limit': 20480.0, 'metric': 'SSD', 'usage': 500.0}], + 'status': 'UP'}])), + ('filter_float_gt', dict( string='objects[?confidence>=0.5].prediction', data={