Merge pull request #124 from nomadbl/master

Update to ProtoBuf 1.0.11

.github/workflows/UnitTest.yml

@@ -13,7 +13,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [macos-latest, ubuntu-latest]
-        julia_version: ["1.3", "1", "nightly"]
+        julia_version: ["1.6", "1", "nightly"]
 
     runs-on: ${{ matrix.os }}
     env:

Project.toml

@@ -1,7 +1,7 @@
 name = "TensorBoardLogger"
 uuid = "899adc3e-224a-11e9-021f-63837185c80f"
 authors = ["Filippo Vicentini <filippovicentini@gmail.com>"]
-version = "0.1.20"
+version = "0.1.30"
 
 [deps]
 CRC32c = "8bf52ea8-c179-5cab-976a-9e18b702a9bc"
@@ -14,10 +14,10 @@ StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 [compat]
 FileIO = "1.2.3"
 ImageCore = "0.8.1, 0.9"
-ProtoBuf = "0.10, 0.11"
+ProtoBuf = "1.0.11"
 Requires = "0.5, 1"
-StatsBase = "0.27, 0.28, 0.29, 0.30, 0.31, 0.32, 0.33"
-julia = "1.3"
+StatsBase = "0.27, 0.28, 0.29, 0.30, 0.31, 0.32, 0.33, 0.34"
+julia = "1.6"
 
 [extras]
 Minio = "4281f0d9-7ae0-406e-9172-b7277c1efa20"

gen/compile_proto.jl

@@ -21,63 +21,45 @@ pbpath =dirname(dirname(PosixPath(pathof(ProtoBuf))))/p"gen"
 cur_path = cwd()
 TBL_root = dirname(cur_path)
 
-src_dir = cur_path/"proto"
-out_dir   = cur_path/"protojl"
+# src_dir = cur_path/"proto"
+src_dir = PosixPath(".")/"proto"
+out_dir = cur_path/"protojl"
 
 ## Clean the output directory
 rm(out_dir, force=true, recursive=true)
 
 ## First module
 function process_module(cur_module::AbstractString; base_module::AbstractString=cur_module, input_path=cur_module)
-    # Include search paths
-    includes   = [src_dir, src_dir/base_module]
 
     # Output folder
-    module_out_dir    = out_dir/cur_module
+    module_out_dir = out_dir/cur_module
 
     # Input files
-    infiles = glob("*.proto", src_dir/input_path)
+    infiles = split.(string.(glob("*.proto", src_dir/input_path)), '/') .|> (a -> a[3:end]) .|> a -> joinpath(a...)
 
     mkpath(module_out_dir)
-    includes_str=["--proto_path=$path" for path=includes]
-    run(ProtoBuf.protoc(`$includes_str --julia_out=$module_out_dir $infiles`))
-
-    nothing
+    relative_paths = string.(infiles)
+    search_directories = joinpath(@__DIR__, "proto")
+    output_directory = string(module_out_dir)
+    # println("relative_paths=$relative_paths")
+    # println("search_directories=$search_directories")
+    # println("output_directory=$output_directory")
+    ProtoBuf.protojl(relative_paths ,search_directories ,output_directory)
+    files_to_include = [string(module_out_dir/basename(file)) for file in infiles]
+    return files_to_include
 end
 
 #process_module("tensorflow", input_path="tensorflow/core/protobuf")
 
-process_module("tensorboard", input_path="tensorboard/compat/proto")
+files_to_include = process_module("tensorboard", input_path="tensorboard/compat/proto")
 
 #plugins = ["audio", "custom_scalar", "hparams", "histogram", "image", "scalar", "text"]
 plugins = ["custom_scalar", "hparams", "text"]
-for plugin in plugins
-    process_module("tensorboard/plugins/$plugin", base_module="tensorboard")
-end
 
+append!(files_to_include, (process_module("tensorboard/plugins/$plugin", base_module="tensorboard") for plugin in plugins)...)
 
-## this fails but would be better
-#cur_module  = "tensorboard"
-#base_module = cur_module
-#
-## Include search paths
-#includes   = [src_dir, src_dir/base_module]
-#
-## Output folder
-#module_out_dir    = out_dir/("$cur_module"*"2")
-#
-## Input files
-#infiles = glob("*.proto", src_dir/cur_module/"compat/proto")
-#
-#for plugin in plugins
-#    plugin_proto_files = glob("*.proto", src_dir/cur_module/"plugins/$plugin")
-#    append!(infiles, plugin_proto_files)
-#end
-#
-#mkpath(module_out_dir)
-#includes_str=["--proto_path=$path" for path=includes]
-#run(ProtoBuf.protoc(`$includes_str --julia_out=$module_out_dir $infiles`))
-
+# files_to_include contains all the proto files, can be used for printing and inspection
+println("generated code for \n$files_to_include")
 
 # Finally move the output directory to the src folder
 mv(out_dir, TBL_root/"src"/"protojl")

src/Deserialization/deserialization.jl

@@ -49,7 +49,7 @@ function read_event(f::IO)
     @assert crc_data == crc_data_ck
 
     pb = PipeBuffer(data)
-    ev = readproto(pb, Event())
+    ev = decode(ProtoDecoder(pb), Event)
     return ev
 end
 
@@ -137,19 +137,9 @@ end
 
 Returns the type of a summary
 """
-function summary_type(summary)
-    if hasproperty(summary, :histo)
-        return :histo
-    elseif hasproperty(summary, :image)
-        return :image
-    elseif hasproperty(summary, :audio)
-        return :audio
-    elseif hasproperty(summary, :tensor)
-        return :tensor
-    #elseif hasproperty(summary, :simple_value)
-    end
-    # always defined
-    return :simple_value
+function summary_type(summary::Summary_Value)
+    summary.value isa Nothing && error("Summary value of Nothing while deserializing")
+    return summary.value.name
 end
 
 """
@@ -193,12 +183,11 @@ SummaryDeserializingIterator(summ; smart=true) =
 function Base.iterate(iter::SummaryDeserializingIterator, state=1)
     evs = iter.summary
     res = iterate(evs, state)
-    res == nothing && return nothing
+    res isa Nothing && return nothing
 
     (tag, summary), i_state = res
 
     typ = summary_type(summary)
-
     if typ === :histo
         val = deserialize_histogram_summary(summary)
         tag, val, state = lookahead_deserialize(tag, val, evs, state, :histo)
@@ -210,7 +199,7 @@ function Base.iterate(iter::SummaryDeserializingIterator, state=1)
     elseif typ === :tensor
         val = deserialize_tensor_summary(summary)
     elseif typ === :simple_value
-        val = summary.simple_value
+        val = summary.value
         tag, val, state = lookahead_deserialize(tag, val, evs, state, :simple_value)
     else
         @error "Event with unknown field" summary=summary
@@ -254,17 +243,15 @@ function map_summaries(fun::Function, logdir; purge=true, tags=nothing, steps=no
 
     for event_file in TBEventFileCollectionIterator(logdir, purge=purge)
         for event in event_file
-            # if event.summary is not defined, don't bother processing this event,
-            # as it's probably a "start file" event or a graph event.
-            !hasproperty(event, :summary) && continue
-
+            # if event.what contains no summary, don't bother processing this event
+            event.what isa Nothing && continue
+            !(event.what.value isa Summary) && continue
             step = event.step
             steps !== nothing && step ∉ steps && continue
 
-            iter = SummaryDeserializingIterator(event.summary, smart)
+            iter = SummaryDeserializingIterator(event.what.value, smart)
             for (name, val) in iter
                 tags !== nothing && name ∉ tags && continue
-
                 fun(name, step, val)
             end
         end

src/Deserialization/histograms.jl

@@ -1,8 +1,9 @@
-function deserialize_histogram_summary(summary)
+function deserialize_histogram_summary(summary::Summary_Value)
     # custom deserialization
     if hasproperty(summary, :metadata)
+        histo = summary.value.value
         if summary.metadata.plugin_data.plugin_name == TB_PLUGIN_JLARRAY_NAME
-            val = reshape(summary.histo.bucket,
+            val = reshape(histo.bucket,
                           reinterpret(Int,
                                       summary.metadata.plugin_data.content)...)
 
@@ -11,7 +12,7 @@ function deserialize_histogram_summary(summary)
     end
 
     # deserialize histogramproto
-    hist_proto = summary.histo
+    hist_proto = summary.value.value
     bin_edges = similar(hist_proto.bucket_limit, length(hist_proto.bucket_limit)+1)
     bin_edges[1] = hist_proto.min
     bin_edges[2:end] .= hist_proto.bucket_limit
@@ -27,7 +28,7 @@ function lookahead_deserialize_histogram_summary(old_tag, old_val, evs::Summary,
     result = old_tag, old_val, state
     # iterate to the next element
     res = iterate(evs, state + 1)
-    res == nothing && return result
+    res isa Nothing && return result
 
     # if the next event is identified, check its type
     (new_tag, summary), i_state = res

src/Deserialization/images.jl

@@ -1,5 +1,5 @@
-function deserialize_image_summary(summary)
-    img = summary.image
+function deserialize_image_summary(summary::Summary_Value)
+    img = summary.value.value
 
     value = load(_format_stream(format"PNG", IOBuffer(img.encoded_image_string)))
 

src/Deserialization/lookahead.jl

@@ -49,19 +49,19 @@ end
 function lookahead_deserialize_simple_value_summary(old_tag, old_val, evs::Summary,
                                                  state)
     # prepare the default output (when no action is taken)
-    result = old_tag, old_val, state
+    result = old_tag, old_val.value, state
     # iterate to the next element
     res = iterate(evs, state + 1)
-    res == nothing && return result
+    res isa Nothing && return result
 
     # if the next event is identified, check its type
     (new_tag, summary), i_state = res
     typ = summary_type(summary)
 
     # if types match, check tags. If they match, return modified structure
     if typ === :simple_value && tags_match_re_im(old_tag, new_tag)
-        val_im = summary.simple_value
-        result = new_tag[1:end-3], old_val + im*val_im, i_state - 1
+        val_im = summary.value.value
+        result = new_tag[1:end-3], old_val.value + im*val_im, i_state - 1
     else
         return result
     end

src/Deserialization/tensor.jl

@@ -1,4 +1,4 @@
-using .tensorboard: TextPluginData
+using .tensorboard_plugin_text: TextPluginData
 
 function deserialize_tensor_summary(summary)
     metadata    = summary.metadata
@@ -15,8 +15,8 @@ function deserialize_tensor_summary(summary)
 end
 
 function deserialize_text_tensor(summary, sizes, names)
-    version_data = readproto(PipeBuffer(summary.metadata.plugin_data.content),
-                             TextPluginData())
+    version_data = decode(ProtoDecoder(PipeBuffer(summary.metadata.plugin_data.content)),
+                             TextPluginData)
     tensor = summary.tensor
     strings = Array{String, length(sizes)}(undef, sizes...)
 

src/Loggers/LogAudio.jl

@@ -1,4 +1,4 @@
-using .tensorboard: Summary_Audio
+using .tensorboard: var"Summary.Audio" as Summary_Audio
 
 """
     log_audios(logger::TBLogger, name::AbstractString, samples::AbstractArray, samplerate::Real; step=step(logger))
@@ -32,6 +32,6 @@ function audio_summary(name::AbstractString, samples::AbstractArray, samplerate:
     io = IOBuffer()
     save(_format_stream(format"WAV",io), samples)
     eas = io.data
-    audio = Summary_Audio(sample_rate = samplerate, num_channels = ndims(samples), length_frames = size(samples, 1), encoded_audio_string = eas, content_type = "audio/wav")
-    Summary_Value(tag=name, audio=audio)
+    audio = Summary_Audio(samplerate, ndims(samples), size(samples, 1), eas, "audio/wav")
+    Summary_Value(name, name, nothing, OneOf(:audio, audio))
 end

src/Loggers/LogHistograms.jl

@@ -48,27 +48,36 @@ end
 function histogram_summary(name::AbstractString, hist::Histogram{T,1}) where T
     edges     = first(hist.edges)
     hist_vals = hist.weights
-
-    hp = HistogramProto(min=first(edges), max=last(edges),
-                        bucket_limit=collect(edges[2:end]),
-                        bucket=hist_vals)
-    return Summary_Value(tag=name, histo=hp)
+    num = length(hist_vals)
+    histsum = sum(hist_vals)
+    histsumsqr = sum(hist_vals.^2) 
+    hp = HistogramProto(first(edges), last(edges),
+                        num,
+                        histsum,
+                        histsumsqr,
+                        collect(edges[2:end]),
+                        hist_vals)
+    return Summary_Value(name, name, nothing, OneOf(:histo, hp))
 end
 
 # Writes to an Histogram summary the flattened version of the array.
 # Also stores the shape of the array as a field in a plugin, which allows to
 # reconstruct the original shape when read back into Julia
 function histogram_arr_summary(name::AbstractString, tensor::AbstractArray)
 
-    smpd = SummaryMetadata_PluginData(plugin_name=TB_PLUGIN_JLARRAY_NAME,
-                    content=reinterpret(UInt8, collect(size(tensor))))
-    sm = SummaryMetadata(plugin_data=smpd)
-
+    smpd = SummaryMetadata_PluginData(TB_PLUGIN_JLARRAY_NAME, reinterpret(UInt8, collect(size(tensor))))
+    sm = SummaryMetadata(smpd, name, "", DataClass.DATA_CLASS_TENSOR)
 
-    edges = collect(0:length(tensor))
-    hp = HistogramProto(min=minimum(edges), max=maximum(edges),
-                        bucket_limit=edges[2:end],
-                        bucket=vec(tensor))
+    num = length(tensor)
+    edges = collect(0:num)
+    histsum = sum(tensor)
+    histsumsqr = sum(tensor.^2) 
+    hp = HistogramProto(minimum(edges), maximum(edges),
+                        num,
+                        histsum,
+                        histsumsqr,
+                        edges[2:end],
+                        vec(tensor))
 
-    return Summary_Value(tag=name, histo=hp, metadata=sm)
+    return Summary_Value(name, name, sm, OneOf(:histo, hp))
 end

src/Loggers/LogImage.jl

@@ -1,13 +1,13 @@
-using .tensorboard: Summary_Image
+using .tensorboard: var"Summary.Image" as Summary_Image
 
 function image_summary(name::AbstractString, img::PngImage)
 
     attr    = img.attr
-    imgsumm = Summary_Image(height     = attr[:height],
-                            width      = attr[:width],
-                            colorspace = attr[:colorspace],
-                            encoded_image_string = img.data )
-    return Summary_Value(tag = name, image = imgsumm)
+    imgsumm = Summary_Image(attr[:height],
+                            attr[:width],
+                            attr[:colorspace],
+                            img.data )
+    return Summary_Value(name, name, nothing, OneOf(:image, imgsumm))
 end