Merge pull request #104 from Juice-jl/compressed-io

Support for `read` and `write` of circuits to *.gz compressed files

Author: guyvdbroeck

.github/workflows/slow_tests.yml

@@ -6,6 +6,7 @@ version = "0.3.1"
 [deps]
 CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
+CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 DirectedAcyclicGraphs = "1e6dae5e-d6e2-422d-9af3-452e7a3785ee"
 Graphs = "86223c79-3864-5bf0-83f7-82e725a168b6"
@@ -23,6 +24,7 @@ TikzGraphs = "b4f28e30-c73f-5eaf-a395-8a9db949a742"
 [compat]
 CSV = "0.7, 0.8, 0.9"
 CUDA = "3"
+CodecZlib = "0.7"
 DataFrames = "1.1, 1.2"
 DirectedAcyclicGraphs = "0.1.1"
 Graphs = "1.4"

Project.toml

@@ -18,9 +18,7 @@ export issomething,
         never, 
         uniform, 
         logsumexp, 
-        noop, 
-        map_values, 
-        groupby
+        noop
 
 
 using CUDA: CuArray, CuVector, CuMatrix, CUDA

src/Utils/misc.jl

@@ -1,7 +1,8 @@
 using LazyArtifacts
 using Lerche: Lerche, Lark, Transformer, @rule, @inline_rule
+using CodecZlib: GzipDecompressorStream, GzipCompressorStream
 
-export FileFormat
+export FileFormat, GzipFormat
 
 # version of model zoo to grab files from
 const zoo_version = "/Circuit-Model-Zoo-0.1.4"
@@ -13,6 +14,10 @@ Lerche.visit_tokens(t::JuiceTransformer) = false
 # file formats supported by this package
 abstract type FileFormat end
 
+struct GzipFormat <: FileFormat 
+    inner_format::FileFormat
+end
+
 # usual comment format for DIMACS-based file formats
 const dimacs_comments = raw"""
     COMMENT : ("c" | "cc") (_WS /[^\n]/*)? (_NL | /$/)
@@ -32,7 +37,11 @@ include("plot.jl")
 # if no logic circuit file format is given on read, infer file format from extension
 
 function file2logicformat(file) 
-    if endswith(file,".jlc")
+    if endswith(file,".gz")
+        file_inner, _ = splitext(file)
+        format_inner = file2logicformat(file_inner)
+        GzipFormat(format_inner)
+    elseif endswith(file,".jlc")
         JlcFormat()
     elseif endswith(file,".sdd")
         SddFormat()
@@ -83,6 +92,10 @@ Base.parse(::Type{LogicCircuit}, args...) =
 Base.read(io::IO, ::Type{LogicCircuit},  args...) = 
     read(io, PlainLogicCircuit,  args...)
 
+Base.read(io::IO, ::Type{LogicCircuit}, f::GzipFormat) = 
+    # avoid method ambiguity
+    read(io, PlainLogicCircuit, f)
+
 #  when asked to parse/read as `StructLogicCircuit`, default to `PlainStructLogicCircuit`
 
 Base.parse(::Type{StructLogicCircuit}, args...) = 
@@ -91,9 +104,17 @@ Base.parse(::Type{StructLogicCircuit}, args...) =
 Base.read(io::IO, ::Type{StructLogicCircuit},  args...) = 
     read(io, PlainStructLogicCircuit,  args...)
 
+Base.read(io::IO, ::Type{StructLogicCircuit}, f::GzipFormat) = 
+    # avoid method ambiguity
+    read(io, PlainStructLogicCircuit, f)
+
 Base.read(ios::Tuple{IO,IO}, ::Type{StructLogicCircuit},  args...) = 
     read(ios, PlainStructLogicCircuit,  args...)
 
+Base.read(ios::Tuple{IO,IO}, ::Type{StructLogicCircuit}, f::Tuple{GzipFormat,VtreeFormat}) = 
+    # avoid method ambiguity
+    read(ios, PlainStructLogicCircuit, f)
+
 # copy read/write API for tuples of files
 
 function Base.read(files::Tuple{AbstractString, AbstractString}, ::Type{C}, args...) where C <: StructLogicCircuit
@@ -111,4 +132,28 @@ function Base.write(files::Tuple{AbstractString,AbstractString},
             write((io1, io2), circuit, args...)
         end
     end
-end
+end
+
+# (de)compress Gzip streams
+
+Base.read(io::IO, circuit_type, f::GzipFormat) =
+    read(GzipDecompressorStream(io), circuit_type, f.inner_format)
+
+Base.write(io::IO, circuit, f::GzipFormat) = begin
+    iogz = GzipCompressorStream(io) 
+    write(iogz, circuit, f.inner_format)
+    close(iogz)
+end
+
+Base.read(ios::Tuple{IO,IO}, circuit_type, f::Tuple{GzipFormat,VtreeFormat}) = begin
+    ios = (GzipDecompressorStream(ios[1]), ios[2])
+    formats = (f[1].inner_format, f[2])
+    read(ios, circuit_type, formats)
+end
+
+Base.write(io::Tuple{IO,IO}, circuit, f::Tuple{GzipFormat,VtreeFormat}) = begin
+    iosgz = (GzipCompressorStream(io[1]), io[2])
+    formats = (f[1].inner_format, f[2])
+    write(iosgz, circuit, formats)
+    close(iosgz[1])
+end

src/io/io.jl

@@ -35,11 +35,16 @@ using LogicCircuits
       cnf_orig = zoo_cnf("easy/C17_mince.cnf")
       @test_nowarn write("$tmp/temp.cnf", cnf_orig)
       cnf_new = read("$tmp/temp.cnf", LogicCircuit)
+
       @test num_variables(cnf_orig) == num_variables(cnf_new)
       @test all(x isa Plain⋁Node || x isa PlainLiteralNode for x in cnf_new.children)
       @test all(all(x isa PlainLiteralNode for x in or_node.children) for or_node in or_nodes(cnf_new))
       @test tree_formula_string(cnf_orig) == tree_formula_string(cnf_new)
 
+      @test_nowarn write("$tmp/temp.cnf.gz", cnf_orig)
+      cnf_new = read("$tmp/temp.cnf.gz", LogicCircuit)  
+      @test tree_formula_string(cnf_orig) == tree_formula_string(cnf_new)
+
   end
 
 end
@@ -66,6 +71,10 @@ end
       circuit2 = read(temp_path, LogicCircuit)
       test8(circuit2)
 
+      write("$temp_path.gz", circuit)
+      circuit2 = read("$temp_path.gz", LogicCircuit)
+      test8(circuit2)
+
   end
 
 end

test/_manual_/aqua_test.jl renamed to test/aqua_test.jl

@@ -17,10 +17,17 @@ include("../helper/little_circuits.jl")
     jlc2 = read(jlc_path, LogicCircuit, JlcFormat())
 
     @test jlc2 isa PlainLogicCircuit
-
     @test num_nodes(sdd) == num_nodes(jlc2)
     @test prob_equiv(sdd, jlc2, 10)
 
+    # write/read as an instructured logic circuit, compressed 
+    write("$jlc_path.gz", sdd)
+    jlc2 = read("$jlc_path.gz", LogicCircuit)
+
+    @test jlc2 isa PlainLogicCircuit
+    @test num_nodes(sdd) == num_nodes(jlc2)
+    @test prob_equiv(sdd, jlc2, 10)
+    
     # write with vtree
     vtree_path = "$tmp/example.vtree"
     paths = (jlc_path, vtree_path)
@@ -31,10 +38,18 @@ include("../helper/little_circuits.jl")
     jlc3 = read(paths, StructLogicCircuit, formats) 
 
     @test jlc3 isa PlainStructLogicCircuit
-
     @test num_nodes(sdd) == num_nodes(jlc3)
     @test prob_equiv(sdd, jlc3, 10)
+    @test Vtree(mgr(sdd)) == vtree(jlc3)
 
+    # write/read with vtree, compressed
+    pathsgz = ("$jlc_path.gz", vtree_path)
+    write(pathsgz, sdd)
+    jlc3 = read(pathsgz, StructLogicCircuit) 
+    
+    @test jlc3 isa PlainStructLogicCircuit
+    @test num_nodes(sdd) == num_nodes(jlc3)
+    @test prob_equiv(sdd, jlc3, 10)
     @test Vtree(mgr(sdd)) == vtree(jlc3)
 
   end

test/io/fnf_io_test.jl

@@ -46,6 +46,14 @@ include("../helper/little_circuits.jl")
 
         @test prob_equiv(circuit, circuit2, 10)
 
+        write("$temp_path.gz", circuit)
+
+        circuit2 = read("$temp_path.gz", LogicCircuit)
+        
+        test_circuit(circuit2)
+
+        @test prob_equiv(circuit, circuit2, 10)
+
     end
 end
 
@@ -62,5 +70,13 @@ end
         @test num_nodes(circuit) == num_nodes(circuit2)
         @test num_edges(circuit) == num_edges(circuit2)
         @test prob_equiv(circuit, circuit2, 10)
+
+        write("$temp_path.gz", circuit)
+
+        circuit2 = read("$temp_path.gz", LogicCircuit)
+                
+        @test num_nodes(circuit) == num_nodes(circuit2)
+        @test num_edges(circuit) == num_edges(circuit2)
+        @test prob_equiv(circuit, circuit2, 10)
     end
 end

test/io/jlc_io_test.jl

@@ -66,6 +66,16 @@ end
 
     @test Vtree(mgr(sdd)) == Vtree(mgr(sdd4))
 
+    # write/read compressed
+    paths = ("$sdd_path.gz", vtree_path)
+    write(paths, sdd)
+    sdd4 = read(paths, StructLogicCircuit) 
+    
+    @test sdd4 isa PlainStructLogicCircuit
+    @test num_nodes(sdd) == num_nodes(sdd4)
+    @test prob_equiv(sdd, sdd4, 10)
+    @test Vtree(mgr(sdd)) == vtree(sdd4)
+    
   end
 
   mktempdir() do tmp

test/io/nnf_io_test.jl

@@ -2,4 +2,4 @@
 
 using Jive
 
-runtests(@__DIR__, skip=["runtests.jl", "helper/", "_manual_/", "build/"])
+runtests(@__DIR__, skip=["runtests.jl", "helper/", "build/"])