Merge pull request #50 from ProgramingIsTheFuture/new_features

davesnx · web-flow · commit 8109d0a8aafa · 2025-05-07T08:57:54.000+02:00
new functions + more tests and fixing range
diff --git a/README.md b/README.md
@@ -125,6 +125,7 @@ The report shows that **query-json** is between 2x and 5x faster than **jq** in
 #### [Builtin operators and functions](https://stedolan.github.io/jq/manual/v1.6/#Builtinoperatorsandfunctions)
 
   - Abs: `abs` ✅
+  - Add: `add` ✅
   - Addition: `+` ✅
   - Subtraction: `-` ✅
   - Multiplication, division, modulo: `*`, `/`, and `%` ✅
diff --git a/source/Ast.ml b/source/Ast.ml
@@ -5,6 +5,8 @@ type literal =
   | Null (* null *)
 [@@deriving show { with_path = false }]
 
+type builtin = Add | Abs [@@deriving show { with_path = false }]
+
 type op =
   | Add
   | Sub
@@ -82,7 +84,6 @@ type expression =
   | Some_ of expression (* some, Some_ to not collide with option *)
   | Find of expression (* find(x) *)
   (* operations *)
-  | Abs
   | Operation of expression * op * expression
   (* Generic *)
   | Length (* length *)
@@ -102,4 +103,6 @@ type expression =
   | Break (* break *)
   (* Conditionals *)
   | Not (* not *)
+  (* builtin *)
+  | Fun of builtin
 [@@deriving show { with_path = false }]
diff --git a/source/Compiler.ml b/source/Compiler.ml
@@ -39,10 +39,9 @@ let make_error (name : string) (json : Json.t) =
   let itemName = get_field_name json in
   make_error_wrong_operation name itemName json
 
-let empty = Ok []
-
 module Output = struct
   let return x = Ok [ x ]
+  let empty = Ok []
 
   let lift2 (f : 'a -> 'b -> 'c) (mx : ('a, string) result)
       (my : ('b, string) result) : ('c, string) result =
@@ -60,6 +59,88 @@ end
 
 let ( let* ) = Output.bind
 
+module Operators = struct
+  let not (json : Json.t) =
+    match json with
+    | `Bool false | `Null -> Output.return (`Bool true)
+    | _ -> Output.return (`Bool false)
+
+  let rec merge_map ~(eq_f : 'a -> 'a -> 'b) ~(f : 'a -> 'b)
+      (cmp : 'a -> 'a -> int) (l1 : 'a list) (l2 : 'a list) : 'b list =
+    match (l1, l2) with
+    | [], l2 -> List.map f l2
+    | l1, [] -> List.map f l1
+    | h1 :: t1, h2 :: t2 ->
+        let r = cmp h1 h2 in
+        if r = 0 then eq_f h1 h2 :: merge_map ~eq_f ~f cmp t1 t2
+        else if r < 0 then f h1 :: merge_map ~eq_f ~f cmp t1 l2
+        else f h2 :: merge_map ~eq_f ~f cmp l1 t2
+
+  let rec add str (left : Json.t) (right : Json.t) :
+      (Json.t list, string) result =
+    match (left, right) with
+    | `Float l, `Float r -> Output.return (`Float (l +. r))
+    | `Int l, `Float r -> Output.return (`Float (float_of_int l +. r))
+    | `Float l, `Int r -> Output.return (`Float (l +. float_of_int r))
+    | `Int l, `Int r ->
+        Output.return (`Float (float_of_int l +. float_of_int r))
+    | `Null, `Int r | `Int r, `Null -> Output.return (`Float (float_of_int r))
+    | `Null, `Float r | `Float r, `Null -> Output.return (`Float r)
+    | `String l, `String r -> Output.return (`String (l ^ r))
+    | `Null, `String r | `String r, `Null -> Output.return (`String r)
+    | `Assoc l, `Assoc r -> (
+        let cmp (key1, _) (key2, _) = String.compare key1 key2 in
+        let eq_f (key, v1) (_, v2) =
+          let* result = add str v1 v2 in
+          Output.return (key, result)
+        in
+        match merge_map ~f:Output.return ~eq_f cmp l r |> Output.collect with
+        | Ok l -> Output.return (`Assoc l)
+        | Error e -> Error e)
+    | `Null, `Assoc r | `Assoc r, `Null -> Output.return (`Assoc r)
+    | `List l, `List r -> Output.return (`List (l @ r))
+    | `Null, `List r | `List r, `Null -> Output.return (`List r)
+    | `Null, `Null -> Output.return `Null
+    | _ -> Error (make_error str left)
+
+  let apply_op str fn (left : Json.t) (right : Json.t) =
+    match (left, right) with
+    | `Float l, `Float r -> Output.return (`Float (fn l r))
+    | `Int l, `Float r -> Output.return (`Float (fn (float_of_int l) r))
+    | `Float l, `Int r -> Output.return (`Float (fn l (float_of_int r)))
+    | `Int l, `Int r ->
+        Output.return (`Float (fn (float_of_int l) (float_of_int r)))
+    | _ -> Error (make_error str left)
+
+  let compare str fn (left : Json.t) (right : Json.t) =
+    match (left, right) with
+    | `Float l, `Float r -> Output.return (`Bool (fn l r))
+    | `Int l, `Float r -> Output.return (`Bool (fn (float_of_int l) r))
+    | `Float l, `Int r -> Output.return (`Bool (fn l (float_of_int r)))
+    | `Int l, `Int r ->
+        Output.return (`Bool (fn (float_of_int l) (float_of_int r)))
+    | _ -> Error (make_error str right)
+
+  let condition (str : string) (fn : bool -> bool -> bool) (left : Json.t)
+      (right : Json.t) =
+    match (left, right) with
+    | `Bool l, `Bool r -> Output.return (`Bool (fn l r))
+    | _ -> Error (make_error str right)
+
+  let gt = compare ">" ( > )
+  let gte = compare ">=" ( >= )
+  let lt = compare "<" ( < )
+  let lte = compare "<=" ( <= )
+  let and_ = condition "and" ( && )
+  let or_ = condition "or" ( || )
+  let eq l r = Output.return (`Bool (l = r))
+  let notEq l r = Output.return (`Bool (l <> r))
+  let add = add "+"
+  let sub = apply_op "-" (fun l r -> l -. r)
+  let mult = apply_op "*" (fun l r -> l *. r)
+  let div = apply_op "/" (fun l r -> l /. r)
+end
+
 let keys (json : Json.t) =
   match json with
   | `Assoc _list ->
@@ -101,15 +182,11 @@ let range ?step from upto =
   let rec range ?(step = 1) start stop =
     if step = 0 then []
     else if (step > 0 && start >= stop) || (step < 0 && start <= stop) then []
-    else start :: range ~step (start + step) stop
+    else `Int start :: range ~step (start + step) stop
   in
   match upto with
-  | None ->
-      List.map (fun i -> Output.return (`Int i)) (range 1 from)
-      |> Output.collect
-  | Some upto ->
-      List.map (fun i -> Output.return (`Int i)) (range ?step from upto)
-      |> Output.collect
+  | None -> Output.return (range 1 from)
+  | Some upto -> Output.return (range ?step from upto)
 
 let split expr json =
   match json with
@@ -142,48 +219,6 @@ let length (json : Json.t) =
   | `List list -> Output.return (`Int (list |> List.length))
   | _ -> Error (make_error "length" json)
 
-let not (json : Json.t) =
-  match json with
-  | `Bool false | `Null -> Output.return (`Bool true)
-  | _ -> Output.return (`Bool false)
-
-let apply str fn (left : Json.t) (right : Json.t) =
-  match (left, right) with
-  | `Float l, `Float r -> Output.return (`Float (fn l r))
-  | `Int l, `Float r -> Output.return (`Float (fn (float_of_int l) r))
-  | `Float l, `Int r -> Output.return (`Float (fn l (float_of_int r)))
-  | `Int l, `Int r ->
-      Output.return (`Float (fn (float_of_int l) (float_of_int r)))
-  | _ -> Error (make_error str left)
-
-let compare str fn (left : Json.t) (right : Json.t) =
-  match (left, right) with
-  | `Float l, `Float r -> Output.return (`Bool (fn l r))
-  | `Int l, `Float r -> Output.return (`Bool (fn (float_of_int l) r))
-  | `Float l, `Int r -> Output.return (`Bool (fn l (float_of_int r)))
-  | `Int l, `Int r ->
-      Output.return (`Bool (fn (float_of_int l) (float_of_int r)))
-  | _ -> Error (make_error str right)
-
-let condition (str : string) (fn : bool -> bool -> bool) (left : Json.t)
-    (right : Json.t) =
-  match (left, right) with
-  | `Bool l, `Bool r -> Output.return (`Bool (fn l r))
-  | _ -> Error (make_error str right)
-
-let gt = compare ">" ( > )
-let gte = compare ">=" ( >= )
-let lt = compare "<" ( < )
-let lte = compare "<=" ( <= )
-let and_ = condition "and" ( && )
-let or_ = condition "or" ( || )
-let eq l r = Output.return (`Bool (l = r))
-let notEq l r = Output.return (`Bool (l <> r))
-let add = apply "+" (fun l r -> l +. r)
-let sub = apply "-" (fun l r -> l -. r)
-let mult = apply "*" (fun l r -> l *. r)
-let div = apply "/" (fun l r -> l /. r)
-
 let filter (fn : Json.t -> bool) (json : Json.t) =
   match json with
   | `List list -> Ok (`List (List.filter fn list))
@@ -288,15 +323,15 @@ let slice (start : int option) (finish : int option) (json : Json.t) =
 
 let iterator (json : Json.t) =
   match json with
-  | `List [] -> empty
+  | `List [] -> Output.empty
   | `List items -> Ok items
   | `Assoc obj -> Ok (List.map snd obj)
   | _ -> Error (make_error "[]" json)
 
 let rec compile expression json : (Json.t list, string) result =
   match expression with
   | Identity -> Output.return json
-  | Empty -> empty
+  | Empty -> Output.empty
   | Keys -> keys json
   | Key key -> member key json
   | Optional expr -> (
@@ -309,9 +344,10 @@ let rec compile expression json : (Json.t list, string) result =
   | Head -> head json
   | Tail -> tail json
   | Length -> length json
-  | Not -> not json
+  | Not -> Operators.not json
   | Map expr -> map expr json
   | Operation (left, op, right) -> (
+      let open Operators in
       match op with
       | Add -> operation left right add json
       | Sub -> operation left right sub json
@@ -337,32 +373,44 @@ let rec compile expression json : (Json.t list, string) result =
   | Select conditional -> (
       let* res = compile conditional json in
       match res with
-      | `Bool b -> ( match b with true -> Output.return json | false -> empty)
+      | `Bool b -> (
+          match b with true -> Output.return json | false -> Output.empty)
       | _ -> Error (make_error "select" res))
   | List exprs ->
       Output.collect (List.map (fun expr -> compile expr json) exprs)
       |> Result.map (fun x -> [ `List x ])
   | Comma (leftR, rightR) ->
       Result.bind (compile leftR json) (fun left ->
           Result.bind (compile rightR json) (fun right -> Ok (left @ right)))
+  | Object [] -> Output.return (`Assoc [])
   | Object list -> handle_objects list json
   | Has e -> (
       match e with
       | Literal ((String _ | Number _) as e) -> has json e
       | _ -> Error (show_expression e ^ " is not allowed"))
   | In e -> in_ json e
-  | Range (from, upto, step) -> range ?step from upto
+  | Range (from, upto, step) -> Result.map List.flatten (range ?step from upto)
   | Reverse -> (
       match json with
       | `List l -> Output.return (`List (List.rev l))
       | _ -> Error (make_error "reverse" json))
   | Split expr -> split expr json
   | Join expr -> join expr json
-  | Abs -> (
+  | Fun Abs -> (
       match json with
       | `Int n -> Output.return (`Int (if n < 0 then -n else n))
       | `Float j -> Output.return (`Float (if j < 0. then -.j else j))
       | _ -> Error (make_error "reverse" json))
+  | Fun Add -> (
+      match json with
+      | `List [] -> Output.return `Null
+      | `List l ->
+          List.fold_left
+            (fun acc el ->
+              let* acc = acc in
+              Operators.add acc el)
+            (Output.return `Null) l
+      | _ -> Error (make_error "add" json))
   | IfThenElse (cond, if_branch, else_branch) -> (
       let* cond = compile cond json in
       match cond with
diff --git a/source/Parser.mly b/source/Parser.mly
@@ -11,7 +11,7 @@
 %token <bool> BOOL
 %token NULL
 %token <string> IDENTIFIER
-%token RANGE ABS
+%token RANGE
 %token IF THEN ELSE ELIF END
 %token DOT
 %token RECURSE
@@ -83,8 +83,8 @@ sequence_expr:
     { e }
 
 %inline operator:
-  | ADD {Add}
   | SUB {Sub}
+  | ADD {Add}
   | MULT {Mult}
   | DIV {Div}
   | EQUAL {Eq}
@@ -122,7 +122,7 @@ term:
     { Literal (Bool b) }
   | NULL
     { Literal(Null) }
-  | RANGE; OPEN_PARENT; nl = separated_nonempty_list(SEMICOLON, NUMBER); CLOSE_PARENT;
+  | RANGE; OPEN_PARENT; nl = separated_nonempty_list(SEMICOLON, number); CLOSE_PARENT;
     {
       let nl = List.map int_of_float nl in
       match nl with
@@ -132,8 +132,6 @@ term:
       | x :: y :: z :: [] -> Range (x, Some y, Some z)
       | _ -> failwith "too many arguments for function range"
     }
-  | ABS
-    { Abs }
   | f = FUNCTION; CLOSE_PARENT;
     { failwith (f ^ "(), should contain a body") }
   | f = FUNCTION; cb = sequence_expr; CLOSE_PARENT;
@@ -196,6 +194,8 @@ term:
       | "nan" -> Nan
       | "is_nan" -> IsNan
       | "not" -> Not
+      | "abs" -> Fun (Abs)
+      | "add" -> Fun (Add)
       (* TODO: remove failwiths once we have implemented the functions *)
       | "if" -> failwith @@ Console.Errors.not_implemented f
       | "then" -> failwith @@ Console.Errors.not_implemented f
diff --git a/source/Tokenizer.ml b/source/Tokenizer.ml
@@ -4,7 +4,7 @@ let dot = [%sedlex.regexp? '.']
 let digit = [%sedlex.regexp? '0' .. '9']
 let number = [%sedlex.regexp? Plus digit, Opt '.', Opt (Plus digit)]
 let space = [%sedlex.regexp? Plus ('\n' | '\t' | ' ')]
-let identifier = [%sedlex.regexp? alphabetic, Star (alphabetic | digit)]
+let identifier = [%sedlex.regexp? (alphabetic | '_'), Star (alphabetic | digit | '_')]
 let not_double_quotes = [%sedlex.regexp? Compl '"']
 
 type token =
@@ -45,7 +45,6 @@ type token =
   | ELSE
   | ELIF
   | END
-  | ABS
   | EOF
 [@@deriving show]
 
@@ -106,7 +105,6 @@ let rec tokenize buf =
   | "else" -> Ok ELSE
   | "elif" -> Ok ELIF
   | "end" -> Ok END
-  | "abs" -> Ok ABS
   | dot -> Ok DOT
   | ".." -> Ok RECURSE
   | '"' -> string buf
diff --git a/test/Test_parse.ml b/test/Test_parse.ml
@@ -85,4 +85,5 @@ let tests =
              ( Literal (Bool false),
                Literal (String "Welcome"),
                Literal (String "Real") ) ));
+    case "map(add)" (Map (Fun Add));
   ]
diff --git a/test/Test_runtime.ml b/test/Test_runtime.ml
diff --git a/test/mapping.t b/test/mapping.t

Original file line number	Diff line number	Diff line change
`@@ -85,4 +85,5 @@ let tests =`
`85`	`85`	`( Literal (Bool false),`
`86`	`86`	`Literal (String "Welcome"),`
`87`	`87`	`Literal (String "Real") ) ));`
	`88`	`+ case "map(add)" (Map (Fun Add));`
`88`	`89`	`]`