From cbb0674bab42a92d525c898acd356e8f1953bd89 Mon Sep 17 00:00:00 2001 From: figroc Date: Sat, 4 Jul 2020 23:47:03 +0800 Subject: [PATCH] upgrade to 2.2 (#8) * upgrade tf to 2.2 * pin protobuf to 3.12.2 --- .travis.yml | 7 +++- README.md | 2 +- VERSION | 2 +- build.gradle | 18 +++++--- crate.toml | 2 +- gradle/golang.gradle | 19 ++++----- gradle/rust.gradle | 10 +---- proto/tensorflow/core/example/example.proto | 10 +++-- proto/tensorflow/core/example/feature.proto | 14 ++++--- .../framework/allocation_description.proto | 5 ++- .../core/framework/attr_value.proto | 10 +++-- .../core/framework/cost_graph.proto | 18 ++++++-- .../tensorflow/core/framework/function.proto | 21 ++++++++-- proto/tensorflow/core/framework/graph.proto | 12 +++--- .../tensorflow/core/framework/node_def.proto | 10 +++-- proto/tensorflow/core/framework/op_def.proto | 2 +- .../core/framework/resource_handle.proto | 15 ++++--- .../core/framework/step_stats.proto | 14 ++++--- proto/tensorflow/core/framework/tensor.proto | 12 +++--- .../core/framework/tensor_description.proto | 12 +++--- .../core/framework/tensor_shape.proto | 2 +- proto/tensorflow/core/framework/types.proto | 2 +- .../tensorflow/core/framework/variable.proto | 3 +- .../tensorflow/core/framework/versions.proto | 5 ++- proto/tensorflow/core/protobuf/cluster.proto | 3 +- proto/tensorflow/core/protobuf/config.proto | 41 +++++++++++++------ proto/tensorflow/core/protobuf/debug.proto | 3 +- .../core/protobuf/error_codes.proto | 3 +- .../tensorflow/core/protobuf/meta_graph.proto | 16 +++++--- .../core/protobuf/named_tensor.proto | 6 ++- .../core/protobuf/rewriter_config.proto | 11 +++-- .../core/protobuf/saved_object_graph.proto | 14 +++---- proto/tensorflow/core/protobuf/saver.proto | 3 +- proto/tensorflow/core/protobuf/struct.proto | 2 + .../protobuf/trackable_object_graph.proto | 5 ++- .../core/protobuf/verifier_config.proto | 3 +- proto/tensorflow_serving/util/status.proto | 4 +- update.sh | 9 ++-- 38 files changed, 212 insertions(+), 138 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1cba28d..4b1ff5a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,11 +11,16 @@ addons: - mono - nuget - pkg-config - - protobuf - rust casks: - dotnet-sdk +before_install: + - | + git -C $(brew --repo homebrew/core) checkout f13bf9adadfe0525c2f4aaeb18b21c636bd7ce1a + brew unlink protobuf && HOMEBREW_NO_AUTO_UPDATE=1 brew install protobuf + git -C $(brew --repo homebrew/core) checkout master + before_cache: - brew cleanup - find /usr/local/Homebrew \! -regex ".+\.git.+" -delete diff --git a/README.md b/README.md index 716354c..4ea0aab 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Grpc tools are needed for building variant packages. See `.travis.yml` for details. -*NOTE: grpc@1.26 and protobuf@3.11 are required* +*NOTE: grpc@1.30 and protobuf@3.12 are required* Target | Command | Outputs | Artifacts -------|-----------------|-------------|---------------------------------------- diff --git a/VERSION b/VERSION index 7ec1d6d..ccbccc3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1.0 +2.2.0 diff --git a/build.gradle b/build.gradle index 3d132a6..7bbfbdb 100644 --- a/build.gradle +++ b/build.gradle @@ -3,7 +3,7 @@ buildscript { mavenCentral() } dependencies { - classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.11" + classpath "com.google.protobuf:protobuf-gradle-plugin:0.8.12" } } @@ -11,8 +11,8 @@ buildscript { ext { vers = [tfs: file("VERSION").text.trim(), jvm: "8", - grpc: "1.26.0", - proto: "3.11.0"] + grpc: "1.30.0", + proto: "3.12.0"] dist = [build: "release", name: [_: project.name, title: project.name.tokenize("-").collect{it.capitalize()}.join(" "), @@ -24,10 +24,10 @@ ext { i == 0 ? ((v as int) + 1) : v }.join("."), node: "1.24.0", - rust: "0.4.6"], + rust: "0.6.0"], proto: [_: vers.proto, - go: "1.3.2", - rust: "2.10.1"]], + go: "1.4.2", + rust: "2.14.0"]], license: "Apache-2.0", author: [_: "Figroc Chen ", id: "figroc", @@ -62,6 +62,12 @@ static def pathOfExecutable(name) { return path.in.text.trim() } +static def resultOfShellCode(cmd) { + def shell = "bash -c '${cmd}'".execute() + shell.waitFor() + return shell.in.text.trim() +} + apply from: "gradle/grpc.gradle" apply from: "gradle/cmake.gradle" apply from: "gradle/java.gradle" diff --git a/crate.toml b/crate.toml index 5128c78..55cf203 100644 --- a/crate.toml +++ b/crate.toml @@ -12,7 +12,7 @@ documentation = "https://docs.rs/${TFSCLIENT_NAME}" categories = ["api-bindings", "science"] [dependencies] -futures = "0.1.25" +futures = "0.3.5" grpcio = "${TFSCLIENT_VERSION_GRPC_RUST}" protobuf = "${TFSCLIENT_VERSION_PROTO_RUST}" diff --git a/gradle/golang.gradle b/gradle/golang.gradle index 1770695..da2e15f 100644 --- a/gradle/golang.gradle +++ b/gradle/golang.gradle @@ -34,17 +34,6 @@ task generateProtoGo { standardOutput file("${source}/go.mod").newOutputStream() commandLine "envsubst" } - fileTree(source){ include "*", "github.com/*" }.visit { fd -> - if (fd.file.file) return - if (fd.path == "github.com") return - copy { - from "${source}/go.mod" - into "${source}/${fd.path}" - filter { line -> - line.startsWith("module") ? "module ${fd.path}" : line - } - } - } } } @@ -124,6 +113,14 @@ task finalizeProtoGo { } fileTree(source){ include "*" }.visit { fd -> if (fd.file.file) return + copy { + from "${source}/go.mod" + into "${source}/${fd.path}" + filter { line -> + line.startsWith("module") ? "module ${fd.path}" : line + } + } + println("generate mod: ${fd.path}/go.mod") file("${source}/go.mod").append("\nreplace ${fd.path} => ./${fd.path}\n") } exec { diff --git a/gradle/rust.gradle b/gradle/rust.gradle index f20909d..f03c961 100644 --- a/gradle/rust.gradle +++ b/gradle/rust.gradle @@ -1,15 +1,9 @@ task installProtocGenRust(type: Exec) { - onlyIf { - pathOfExecutable("protoc-gen-rust") == "" - } - commandLine "cargo", "install", "protobuf-codegen" + commandLine "cargo", "install", "--vers", dist.version.proto.rust, "protobuf-codegen" } task installGrpcRustPlugin(type: Exec) { - onlyIf { - pathOfExecutable("grpc_rust_plugin") == "" - } - commandLine "cargo", "install", "grpcio-compiler" + commandLine "cargo", "install", "--vers", dist.version.grpc.rust, "grpcio-compiler" } task generateProtoRust { diff --git a/proto/tensorflow/core/example/example.proto b/proto/tensorflow/core/example/example.proto index e36e51d..32a1d0b 100644 --- a/proto/tensorflow/core/example/example.proto +++ b/proto/tensorflow/core/example/example.proto @@ -2,13 +2,15 @@ // model training or inference. syntax = "proto3"; +package tensorflow; + import "tensorflow/core/example/feature.proto"; + option cc_enable_arenas = true; option java_outer_classname = "ExampleProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.example"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example"; -package tensorflow; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example/example_protos_go_proto"; // An Example is a mostly-normalized data format for storing data for // training and inference. It contains a key-value store (features); where @@ -87,7 +89,7 @@ package tensorflow; message Example { Features features = 1; -}; +} // A SequenceExample is an Example representing one or more sequences, and // some context. The context contains features which apply to the entire @@ -298,4 +300,4 @@ message Example { message SequenceExample { Features context = 1; FeatureLists feature_lists = 2; -}; +} diff --git a/proto/tensorflow/core/example/feature.proto b/proto/tensorflow/core/example/feature.proto index 6d81974..a9496fb 100644 --- a/proto/tensorflow/core/example/feature.proto +++ b/proto/tensorflow/core/example/feature.proto @@ -54,12 +54,14 @@ // syntax = "proto3"; + +package tensorflow; + option cc_enable_arenas = true; option java_outer_classname = "FeatureProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.example"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example"; -package tensorflow; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/example/example_protos_go_proto"; // Containers to hold repeated fundamental values. message BytesList { @@ -80,12 +82,12 @@ message Feature { FloatList float_list = 2; Int64List int64_list = 3; } -}; +} message Features { // Map from feature name to feature. map feature = 1; -}; +} // Containers for sequential data. // @@ -97,9 +99,9 @@ message Features { // message FeatureList { repeated Feature feature = 1; -}; +} message FeatureLists { // Map from feature name to feature list. map feature_list = 1; -}; +} diff --git a/proto/tensorflow/core/framework/allocation_description.proto b/proto/tensorflow/core/framework/allocation_description.proto index 64133b0..f18caa4 100644 --- a/proto/tensorflow/core/framework/allocation_description.proto +++ b/proto/tensorflow/core/framework/allocation_description.proto @@ -1,11 +1,12 @@ syntax = "proto3"; package tensorflow; + option cc_enable_arenas = true; option java_outer_classname = "AllocationDescriptionProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/allocation_description_go_proto"; message AllocationDescription { // Total number of bytes requested @@ -25,4 +26,4 @@ message AllocationDescription { // Address of the allocation. uint64 ptr = 6; -}; +} diff --git a/proto/tensorflow/core/framework/attr_value.proto b/proto/tensorflow/core/framework/attr_value.proto index 054e3ec..2e91313 100644 --- a/proto/tensorflow/core/framework/attr_value.proto +++ b/proto/tensorflow/core/framework/attr_value.proto @@ -1,14 +1,16 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/tensor.proto"; +import "tensorflow/core/framework/tensor_shape.proto"; +import "tensorflow/core/framework/types.proto"; + option cc_enable_arenas = true; option java_outer_classname = "AttrValueProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensorflow/core/framework/tensor.proto"; -import "tensorflow/core/framework/tensor_shape.proto"; -import "tensorflow/core/framework/types.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/attr_value_go_proto"; // Protocol buffer representing the value for an attr used to configure an Op. // Comment indicates the corresponding attr type. Only the field matching the diff --git a/proto/tensorflow/core/framework/cost_graph.proto b/proto/tensorflow/core/framework/cost_graph.proto index cc6bc84..42c9e23 100644 --- a/proto/tensorflow/core/framework/cost_graph.proto +++ b/proto/tensorflow/core/framework/cost_graph.proto @@ -1,13 +1,15 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/tensor_shape.proto"; +import "tensorflow/core/framework/types.proto"; + option cc_enable_arenas = true; option java_outer_classname = "CostGraphProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensorflow/core/framework/tensor_shape.proto"; -import "tensorflow/core/framework/types.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/cost_graph_go_proto"; message CostGraphDef { message Node { @@ -74,4 +76,14 @@ message CostGraphDef { bool inaccurate = 17; } repeated Node node = 1; + + // Total cost of this graph, typically used for balancing decisions. + message AggregatedCost { + // Aggregated cost value. + float cost = 1; + + // Aggregated cost dimension (e.g. 'memory', 'compute', 'network'). + string dimension = 2; + } + repeated AggregatedCost cost = 2; } diff --git a/proto/tensorflow/core/framework/function.proto b/proto/tensorflow/core/framework/function.proto index 7b5756e..6d43c05 100644 --- a/proto/tensorflow/core/framework/function.proto +++ b/proto/tensorflow/core/framework/function.proto @@ -1,14 +1,16 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/attr_value.proto"; +import "tensorflow/core/framework/node_def.proto"; +import "tensorflow/core/framework/op_def.proto"; + option cc_enable_arenas = true; option java_outer_classname = "FunctionProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensorflow/core/framework/attr_value.proto"; -import "tensorflow/core/framework/node_def.proto"; -import "tensorflow/core/framework/op_def.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/function_go_proto"; // A library is a set of named functions. message FunctionDefLibrary { @@ -37,6 +39,17 @@ message FunctionDef { } map arg_attr = 7; + // Unique IDs for each resource argument, used to track aliasing resources. If + // Argument A and Argument B alias each other, then + // resource_arg_unique_ids[A.index] == resource_arg_unique_ids[B.index]. + // + // If this field is empty, none of the arguments could alias; otherwise, every + // resource argument should have an entry in this field. + // + // When instantiated, the unique IDs will be attached to the _Arg nodes' + // "_resource_arg_unique_id" attribute. + map resource_arg_unique_id = 8; + // NOTE: field id 2 deleted on Jan 11, 2017, GraphDef version 21. reserved 2; diff --git a/proto/tensorflow/core/framework/graph.proto b/proto/tensorflow/core/framework/graph.proto index 76d3589..f49e41a 100644 --- a/proto/tensorflow/core/framework/graph.proto +++ b/proto/tensorflow/core/framework/graph.proto @@ -1,14 +1,16 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/function.proto"; +import "tensorflow/core/framework/node_def.proto"; +import "tensorflow/core/framework/versions.proto"; + option cc_enable_arenas = true; option java_outer_classname = "GraphProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensorflow/core/framework/node_def.proto"; -import "tensorflow/core/framework/function.proto"; -import "tensorflow/core/framework/versions.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/graph_go_proto"; // Represents the graph of operations message GraphDef { @@ -53,4 +55,4 @@ message GraphDef { // consumer does not start until all return values of the callee // function are ready. FunctionDefLibrary library = 2; -}; +} diff --git a/proto/tensorflow/core/framework/node_def.proto b/proto/tensorflow/core/framework/node_def.proto index 3c89f78..c0dc683 100644 --- a/proto/tensorflow/core/framework/node_def.proto +++ b/proto/tensorflow/core/framework/node_def.proto @@ -1,12 +1,14 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/attr_value.proto"; + option cc_enable_arenas = true; option java_outer_classname = "NodeProto"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensorflow/core/framework/attr_value.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/node_def_go_proto"; message NodeDef { // The name given to this operator. Used for naming inputs, @@ -79,8 +81,8 @@ message NodeDef { // `original_node_names` can be used to map errors originating at the // current ndoe to some top level source code. repeated string original_func_names = 2; - }; + } // This stores debug information associated with the node. ExperimentalDebugInfo experimental_debug_info = 6; -}; +} diff --git a/proto/tensorflow/core/framework/op_def.proto b/proto/tensorflow/core/framework/op_def.proto index 9b65e23..ad109a3 100644 --- a/proto/tensorflow/core/framework/op_def.proto +++ b/proto/tensorflow/core/framework/op_def.proto @@ -5,7 +5,7 @@ option cc_enable_arenas = true; option java_outer_classname = "OpDefProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/op_def_go_proto"; import "tensorflow/core/framework/attr_value.proto"; import "tensorflow/core/framework/types.proto"; diff --git a/proto/tensorflow/core/framework/resource_handle.proto b/proto/tensorflow/core/framework/resource_handle.proto index 4a03fc7..eb0d163 100644 --- a/proto/tensorflow/core/framework/resource_handle.proto +++ b/proto/tensorflow/core/framework/resource_handle.proto @@ -1,14 +1,15 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/tensor_shape.proto"; +import "tensorflow/core/framework/types.proto"; + option cc_enable_arenas = true; option java_outer_classname = "ResourceHandle"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; - -import "tensorflow/core/framework/tensor_shape.proto"; -import "tensorflow/core/framework/types.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/resource_handle_go_proto"; // Protocol buffer representing a handle to a tensorflow resource. Handles are // not valid across executions, but can be serialized back and forth from within @@ -39,4 +40,8 @@ message ResourceHandleProto { // Data types and shapes for the underlying resource. repeated DtypeAndShape dtypes_and_shapes = 6; -}; + + // A set of devices containing the resource. If empty, the resource only + // exists on `device`. + repeated string allowed_devices = 7; +} diff --git a/proto/tensorflow/core/framework/step_stats.proto b/proto/tensorflow/core/framework/step_stats.proto index f8cab13..762487f 100644 --- a/proto/tensorflow/core/framework/step_stats.proto +++ b/proto/tensorflow/core/framework/step_stats.proto @@ -1,13 +1,15 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/allocation_description.proto"; +import "tensorflow/core/framework/tensor_description.proto"; + option cc_enable_arenas = true; option java_outer_classname = "StepStatsProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensorflow/core/framework/allocation_description.proto"; -import "tensorflow/core/framework/tensor_description.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/step_stats_go_proto"; // An allocation/de-allocation operation performed by the allocator. message AllocationRecord { @@ -36,7 +38,7 @@ message AllocatorMemoryUsed { message NodeOutput { int32 slot = 1; TensorDescription tensor_description = 3; -}; +} // For memory tracking. message MemoryStats { @@ -72,7 +74,7 @@ message NodeExecStats { int64 op_end_rel_nanos = 15; int64 all_end_rel_nanos = 16; int64 scheduled_nanos = 17; -}; +} message DeviceStepStats { string device = 1; @@ -83,4 +85,4 @@ message DeviceStepStats { message StepStats { repeated DeviceStepStats dev_stats = 1; -}; +} diff --git a/proto/tensorflow/core/framework/tensor.proto b/proto/tensorflow/core/framework/tensor.proto index 55921af..47b694f 100644 --- a/proto/tensorflow/core/framework/tensor.proto +++ b/proto/tensorflow/core/framework/tensor.proto @@ -1,14 +1,16 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/resource_handle.proto"; +import "tensorflow/core/framework/tensor_shape.proto"; +import "tensorflow/core/framework/types.proto"; + option cc_enable_arenas = true; option java_outer_classname = "TensorProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensorflow/core/framework/resource_handle.proto"; -import "tensorflow/core/framework/tensor_shape.proto"; -import "tensorflow/core/framework/types.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_go_proto"; // Protocol buffer representing a tensor. message TensorProto { @@ -81,7 +83,7 @@ message TensorProto { // DT_UINT64 repeated uint64 uint64_val = 17 [packed = true]; -}; +} // Protocol buffer representing the serialization format of DT_VARIANT tensors. message VariantTensorDataProto { diff --git a/proto/tensorflow/core/framework/tensor_description.proto b/proto/tensorflow/core/framework/tensor_description.proto index 4c23c7e..8462c63 100644 --- a/proto/tensorflow/core/framework/tensor_description.proto +++ b/proto/tensorflow/core/framework/tensor_description.proto @@ -1,14 +1,16 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/allocation_description.proto"; +import "tensorflow/core/framework/tensor_shape.proto"; +import "tensorflow/core/framework/types.proto"; + option cc_enable_arenas = true; option java_outer_classname = "TensorDescriptionProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; -import "tensorflow/core/framework/types.proto"; -import "tensorflow/core/framework/tensor_shape.proto"; -import "tensorflow/core/framework/allocation_description.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_description_go_proto"; message TensorDescription { // Data type of tensor elements @@ -19,4 +21,4 @@ message TensorDescription { // Information about the size and allocator used for the data AllocationDescription allocation_description = 4; -}; +} diff --git a/proto/tensorflow/core/framework/tensor_shape.proto b/proto/tensorflow/core/framework/tensor_shape.proto index 286156a..45d5b78 100644 --- a/proto/tensorflow/core/framework/tensor_shape.proto +++ b/proto/tensorflow/core/framework/tensor_shape.proto @@ -5,7 +5,7 @@ option cc_enable_arenas = true; option java_outer_classname = "TensorShapeProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/tensor_shape_go_proto"; package tensorflow; diff --git a/proto/tensorflow/core/framework/types.proto b/proto/tensorflow/core/framework/types.proto index 5356f9f..900132c 100644 --- a/proto/tensorflow/core/framework/types.proto +++ b/proto/tensorflow/core/framework/types.proto @@ -5,7 +5,7 @@ option cc_enable_arenas = true; option java_outer_classname = "TypesProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/types_go_proto"; // (== suppress_warning documentation-presence ==) // LINT.IfChange diff --git a/proto/tensorflow/core/framework/variable.proto b/proto/tensorflow/core/framework/variable.proto index b2978c7..09d7fb3 100644 --- a/proto/tensorflow/core/framework/variable.proto +++ b/proto/tensorflow/core/framework/variable.proto @@ -6,8 +6,7 @@ option cc_enable_arenas = true; option java_outer_classname = "VariableProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; - -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/variable_go_proto"; // Indicates when a distributed variable will be synced. enum VariableSynchronization { diff --git a/proto/tensorflow/core/framework/versions.proto b/proto/tensorflow/core/framework/versions.proto index dd2ec55..2cca6e3 100644 --- a/proto/tensorflow/core/framework/versions.proto +++ b/proto/tensorflow/core/framework/versions.proto @@ -1,11 +1,12 @@ syntax = "proto3"; package tensorflow; + option cc_enable_arenas = true; option java_outer_classname = "VersionsProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework/versions_go_proto"; // Version information for a piece of serialized data // @@ -29,4 +30,4 @@ message VersionDef { // Specific consumer versions which are disallowed (e.g. due to bugs). repeated int32 bad_consumers = 3; -}; +} diff --git a/proto/tensorflow/core/protobuf/cluster.proto b/proto/tensorflow/core/protobuf/cluster.proto index c696d34..6c7162f 100644 --- a/proto/tensorflow/core/protobuf/cluster.proto +++ b/proto/tensorflow/core/protobuf/cluster.proto @@ -16,11 +16,12 @@ limitations under the License. syntax = "proto3"; package tensorflow; + option cc_enable_arenas = true; option java_outer_classname = "ClusterProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.distruntime"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; // This file contains protos to be used when defining a TensorFlow // cluster. diff --git a/proto/tensorflow/core/protobuf/config.proto b/proto/tensorflow/core/protobuf/config.proto index bce52c6..93f350f 100644 --- a/proto/tensorflow/core/protobuf/config.proto +++ b/proto/tensorflow/core/protobuf/config.proto @@ -2,12 +2,6 @@ syntax = "proto3"; package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "ConfigProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; - -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; import "tensorflow/core/framework/cost_graph.proto"; import "tensorflow/core/framework/graph.proto"; import "tensorflow/core/framework/step_stats.proto"; @@ -15,6 +9,12 @@ import "tensorflow/core/protobuf/cluster.proto"; import "tensorflow/core/protobuf/debug.proto"; import "tensorflow/core/protobuf/rewriter_config.proto"; +option cc_enable_arenas = true; +option java_outer_classname = "ConfigProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; + message GPUOptions { // Fraction of the available GPU memory to allocate for each process. // 1 means to allocate all of the GPU memory, 0.5 means the process @@ -367,6 +367,17 @@ message ConfigProto { // The execution of an individual op (for some op types) can be // parallelized on a pool of intra_op_parallelism_threads. // 0 means the system picks an appropriate number. + // + // If you create an ordinary session, e.g., from Python or C++, + // then there is exactly one intra op thread pool per process. + // The first session created determines the number of threads in this pool. + // All subsequent sessions reuse/share this one global pool. + // + // There are notable exceptions to the default behavior describe above: + // 1. There is an environment variable for overriding this thread pool, + // named TF_OVERRIDE_GLOBAL_THREADPOOL. + // 2. When connecting to a server, such as a remote `tf.train.Server` + // instance, then this option will be ignored altogether. int32 intra_op_parallelism_threads = 2; // Nodes that perform blocking operations are enqueued on a pool of @@ -456,6 +467,12 @@ message ConfigProto { // enabled, this field is ignored and sessions are always isolated. bool isolate_session_state = 15; + // When true, WorkerSessions are created with device attributes from the + // full cluster. + // This is helpful when a worker wants to partition a graph + // (for example during a PartitionedCallOp). + bool share_cluster_devices_in_session = 17; + // Everything inside Experimental is subject to change and is not subject // to API stability guarantees in // https://www.tensorflow.org/guide/version_compat. @@ -519,10 +536,8 @@ message ConfigProto { // CPU usage. bool disable_thread_spinning = 9; - // When true, WorkerSessions are created with device attributes from the - // full cluster. - // This is helpful when a worker wants to partition a graph - // (for example during a PartitionedCallOp). + // This was promoted to a non-experimental API. Please use + // ConfigProto.share_cluster_devices_in_session instead. bool share_cluster_devices_in_session = 10; // Metadata about the session. @@ -567,11 +582,11 @@ message ConfigProto { // The XLA fusion autotuner can improve performance by executing a heuristic // search on the compiler parameters. int64 xla_fusion_autotuner_thresh = 15; - }; + } Experimental experimental = 16; - // Next: 17 + // Next: 18 } // Options for a single Run() call. @@ -625,7 +640,7 @@ message RunOptions { // and tail) latency. // Consider using this option for CPU-bound workloads like inference. bool use_run_handler_pool = 2; - }; + } Experimental experimental = 8; diff --git a/proto/tensorflow/core/protobuf/debug.proto b/proto/tensorflow/core/protobuf/debug.proto index 3cfab17..21df4a1 100644 --- a/proto/tensorflow/core/protobuf/debug.proto +++ b/proto/tensorflow/core/protobuf/debug.proto @@ -1,11 +1,12 @@ syntax = "proto3"; package tensorflow; + option cc_enable_arenas = true; option java_outer_classname = "DebugProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; // Option for watching a node in TensorFlow Debugger (tfdbg). message DebugTensorWatch { diff --git a/proto/tensorflow/core/protobuf/error_codes.proto b/proto/tensorflow/core/protobuf/error_codes.proto index 9ef59d7..a880fdf 100644 --- a/proto/tensorflow/core/protobuf/error_codes.proto +++ b/proto/tensorflow/core/protobuf/error_codes.proto @@ -6,8 +6,7 @@ option cc_enable_arenas = true; option java_outer_classname = "ErrorCodesProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; - -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; // The canonical error codes for TensorFlow APIs. // diff --git a/proto/tensorflow/core/protobuf/meta_graph.proto b/proto/tensorflow/core/protobuf/meta_graph.proto index 1eb2023..c560451 100644 --- a/proto/tensorflow/core/protobuf/meta_graph.proto +++ b/proto/tensorflow/core/protobuf/meta_graph.proto @@ -1,13 +1,8 @@ syntax = "proto3"; package tensorflow; -option cc_enable_arenas = true; -option java_outer_classname = "MetaGraphProtos"; -option java_multiple_files = true; -option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; -import "google/protobuf/any.proto"; +import "google/protobuf/any.proto"; import "tensorflow/core/framework/graph.proto"; import "tensorflow/core/framework/op_def.proto"; import "tensorflow/core/framework/tensor_shape.proto"; @@ -16,6 +11,12 @@ import "tensorflow/core/protobuf/saved_object_graph.proto"; import "tensorflow/core/protobuf/saver.proto"; import "tensorflow/core/protobuf/struct.proto"; +option cc_enable_arenas = true; +option java_outer_classname = "MetaGraphProtos"; +option java_multiple_files = true; +option java_package = "org.tensorflow.framework"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; + // NOTE: This protocol buffer is evolving, and will go through revisions in the // coming months. // @@ -67,6 +68,9 @@ message MetaGraphDef { // A flag to denote whether default-valued attrs have been stripped from // the nodes in this graph_def. bool stripped_default_attrs = 7; + + // FunctionDef name to aliases mapping. + map function_aliases = 8; } MetaInfoDef meta_info_def = 1; diff --git a/proto/tensorflow/core/protobuf/named_tensor.proto b/proto/tensorflow/core/protobuf/named_tensor.proto index 6e2f7fe..4210f04 100644 --- a/proto/tensorflow/core/protobuf/named_tensor.proto +++ b/proto/tensorflow/core/protobuf/named_tensor.proto @@ -1,12 +1,14 @@ syntax = "proto3"; package tensorflow; + +import "tensorflow/core/framework/tensor.proto"; + option cc_enable_arenas = true; option java_outer_classname = "NamedTensorProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; -import "tensorflow/core/framework/tensor.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; // A pair of tensor name and tensor values. message NamedTensorProto { diff --git a/proto/tensorflow/core/protobuf/rewriter_config.proto b/proto/tensorflow/core/protobuf/rewriter_config.proto index 54943ee..e657a18 100644 --- a/proto/tensorflow/core/protobuf/rewriter_config.proto +++ b/proto/tensorflow/core/protobuf/rewriter_config.proto @@ -2,15 +2,14 @@ syntax = "proto3"; package tensorflow; +import "tensorflow/core/framework/attr_value.proto"; +import "tensorflow/core/protobuf/verifier_config.proto"; + option cc_enable_arenas = true; option java_outer_classname = "RewriterConfigProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; - -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; - -import "tensorflow/core/framework/attr_value.proto"; -import "tensorflow/core/protobuf/verifier_config.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; message AutoParallelOptions { bool enable = 1; @@ -130,7 +129,7 @@ message RewriterConfig { // effect on manually requested memory optimization passes in the optimizers // field. MemOptType memory_optimization = 4; - // A node name scope for node names which are valid outputs of recompuations. + // A node name scope for node names which are valid outputs of recomputations. // Inputs to nodes that match this scope may be recomputed (subject either to // manual annotation of those input nodes or to manual annotation and // heuristics depending on memory_optimization), but the nodes themselves will diff --git a/proto/tensorflow/core/protobuf/saved_object_graph.proto b/proto/tensorflow/core/protobuf/saved_object_graph.proto index 20074f3..e794b88 100644 --- a/proto/tensorflow/core/protobuf/saved_object_graph.proto +++ b/proto/tensorflow/core/protobuf/saved_object_graph.proto @@ -1,15 +1,16 @@ syntax = "proto3"; -import "tensorflow/core/protobuf/trackable_object_graph.proto"; -import "tensorflow/core/protobuf/struct.proto"; +package tensorflow; + import "tensorflow/core/framework/tensor_shape.proto"; import "tensorflow/core/framework/types.proto"; -import "tensorflow/core/framework/versions.proto"; import "tensorflow/core/framework/variable.proto"; +import "tensorflow/core/framework/versions.proto"; +import "tensorflow/core/protobuf/struct.proto"; +import "tensorflow/core/protobuf/trackable_object_graph.proto"; option cc_enable_arenas = true; - -package tensorflow; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; // A SavedObjectGraph is part of object-based SavedModels in TF 2.0. It // describes the directed graph of Python objects (or equivalent in other @@ -37,8 +38,7 @@ message SavedObject { // graph. // // Note: currently only valid if kind == "user_object". - repeated TrackableObjectGraph.TrackableObject.ObjectReference - children = 1; + repeated TrackableObjectGraph.TrackableObject.ObjectReference children = 1; // Removed when forking SavedObject from TrackableObjectGraph. reserved "attributes"; diff --git a/proto/tensorflow/core/protobuf/saver.proto b/proto/tensorflow/core/protobuf/saver.proto index 4245386..5ba7944 100644 --- a/proto/tensorflow/core/protobuf/saver.proto +++ b/proto/tensorflow/core/protobuf/saver.proto @@ -1,11 +1,12 @@ syntax = "proto3"; package tensorflow; + option cc_enable_arenas = true; option java_outer_classname = "SaverProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.util"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; // Protocol buffer representing the configuration of a Saver. message SaverDef { diff --git a/proto/tensorflow/core/protobuf/struct.proto b/proto/tensorflow/core/protobuf/struct.proto index 8983db0..91af120 100644 --- a/proto/tensorflow/core/protobuf/struct.proto +++ b/proto/tensorflow/core/protobuf/struct.proto @@ -5,6 +5,8 @@ package tensorflow; import "tensorflow/core/framework/tensor_shape.proto"; import "tensorflow/core/framework/types.proto"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; + // `StructuredValue` represents a dynamically typed value representing various // data structures that are inspired by Python data structures typically used in // TensorFlow functions as inputs and outputs. diff --git a/proto/tensorflow/core/protobuf/trackable_object_graph.proto b/proto/tensorflow/core/protobuf/trackable_object_graph.proto index 02d852e..48dbd92 100644 --- a/proto/tensorflow/core/protobuf/trackable_object_graph.proto +++ b/proto/tensorflow/core/protobuf/trackable_object_graph.proto @@ -1,9 +1,10 @@ syntax = "proto3"; -option cc_enable_arenas = true; - package tensorflow; +option cc_enable_arenas = true; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; + // A TensorBundle addition which saves extra information about the objects which // own variables, allowing for more robust checkpoint loading into modified // programs. diff --git a/proto/tensorflow/core/protobuf/verifier_config.proto b/proto/tensorflow/core/protobuf/verifier_config.proto index 5a1373b..4440aad 100644 --- a/proto/tensorflow/core/protobuf/verifier_config.proto +++ b/proto/tensorflow/core/protobuf/verifier_config.proto @@ -1,11 +1,12 @@ syntax = "proto3"; package tensorflow; + option cc_enable_arenas = true; option java_outer_classname = "VerifierConfigProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/protobuf"; +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/core_protos_go_proto"; // The config for graph verifiers. message VerifierConfig { diff --git a/proto/tensorflow_serving/util/status.proto b/proto/tensorflow_serving/util/status.proto index c3f863c..332d1b7 100644 --- a/proto/tensorflow_serving/util/status.proto +++ b/proto/tensorflow_serving/util/status.proto @@ -1,10 +1,10 @@ syntax = "proto3"; -option cc_enable_arenas = true; +package tensorflow.serving; import "tensorflow/core/protobuf/error_codes.proto"; -package tensorflow.serving; +option cc_enable_arenas = true; // Status that corresponds to Status in // third_party/tensorflow/core/lib/core/status.h. diff --git a/update.sh b/update.sh index 59f18bc..84fd35a 100755 --- a/update.sh +++ b/update.sh @@ -9,20 +9,19 @@ goto=proto apis=tensorflow_serving/apis rm -rf ${repo} -function fetchRepo { - set -e +function fetchRepo {( set -e mkdir -p ${repo}/tmp git -C ${repo}/tmp \ clone --depth 1 -b ${ver} \ https://github.com/tensorflow/${1}.git mv ${repo}/tmp/${1}/${2} ${repo}/ rm -rf ${repo}/tmp -} +)} fetchRepo tensorflow tensorflow && echo fetchRepo serving tensorflow_serving && echo rm -rf ${goto} -function importProto { +function importProto {( set -e if [[ ! -f ${goto}/${1} ]]; then echo "${1}" mkdir -p ${goto}/${1%/*} @@ -36,7 +35,7 @@ function importProto { fi done fi -} +)} for i in ${repo}/${apis}/*_service.proto; do importProto ${i#${repo}/} done