Merge remote-tracking branch 'upstream/main' into anton/7392/fix-pyth…

…on-test
protocolbuffers · Mar 9, 2024 · 06c8485 · 06c8485
2 parents 277e212 + 04397d4
commit 06c8485
Show file tree

Hide file tree

Showing 236 changed files with 18,452 additions and 3,618 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -30,3 +30,5 @@ common --noenable_bzlmod
 # Important: this flag ensures that we remain compliant with the C++ layering
 # check.
 build --features=layering_check
+
+common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1
diff --git a/.github/BUILD.bazel b/.github/BUILD.bazel
@@ -11,7 +11,7 @@ xcode_version(
     default_macos_sdk_version = "13.1",
     default_tvos_sdk_version = "16.1",
     default_watchos_sdk_version = "9.1",
-    version = "14.2.14C18",
+    version = "14.2.0.14C18",
 )
 
 xcode_version(
@@ -26,7 +26,7 @@ xcode_version(
 
 xcode_config(
     name = "host_xcodes",
-    default = ":version14_1_0_14B47b",
+    default = ":version14_2_14C18",
     versions = [
         ":version14_2_14C18",
         ":version14_1_0_14B47b",

diff --git a/.github/workflows/staleness_check.yml b/.github/workflows/staleness_check.yml
@@ -55,9 +55,9 @@ jobs:
           bazel-cache: staleness
           bash: >
             set -ex;
-            if [[ -z $COMMIT_TRIGGERED_RUN || -z $MAIN_RUN ]]; then
-            bazel query 'attr(tags, "staleness_test", //...)' | xargs bazel test $BAZEL_FLAGS ||
             echo "Please run ./regenerate_stale_files.sh to regenerate stale files";
+            if [[ -z $COMMIT_TRIGGERED_RUN || -z $MAIN_RUN ]]; then
+            bazel query 'attr(tags, "staleness_test", //...)' | xargs bazel test $BAZEL_FLAGS;
             else
             bazel query 'attr(tags, "staleness_test", //...)';
             fi
diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml
@@ -320,12 +320,11 @@ jobs:
             os: macos-12
             cache_key: macos-12
             bazel: test //src/... //third_party/utf8_range/...
-          # TODO Enable these once mac setup is working for Bazel 7
-          #- name: MacOS Bazel 7
-          #  os: macos-12
-          #  cache_key: macos-12-bazel7
-          #  bazel: test //src/... //third_party/utf8_range/...
-          #  bazel_version: '7.0.0'
+          - name: MacOS Bazel 7
+            os: macos-12
+            cache_key: macos-12-bazel7
+            bazel: test //src/... //third_party/utf8_range/...
+            bazel_version: '7.0.0'
           - name: MacOS Apple Silicon (build only) Bazel
             os: macos-12
             cache_key: macos-12-arm

diff --git a/.github/workflows/test_csharp.yml b/.github/workflows/test_csharp.yml
@@ -20,13 +20,31 @@ jobs:
         uses: protocolbuffers/protobuf-ci/checkout@v2
         with:
           ref: ${{ inputs.safe-checkout }}
+
+      # TODO Run this with Bazel once codegen is handled properly.
       - name: Run tests
+        uses: protocolbuffers/protobuf-ci/docker@v2
+        with:
+          image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          entrypoint: /bin/bash
+          command: >-
+            -c "
+            cd csharp &&
+            dotnet restore src/Google.Protobuf.sln &&
+            dotnet build -c Release src/Google.Protobuf.sln &&
+            dotnet test -c Release -f net6.0 src/Google.Protobuf.Test/Google.Protobuf.Test.csproj"
+
+      - name: Clear bazel between docker instances
+        run: sudo rm -rf _build
+
+      - name: Run conformance tests
         uses: protocolbuffers/protobuf-ci/bazel-docker@v2
         with:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/csharp:3.1.415-6.0.100-66964dc8b07b6d1fc73a5cc14e59e84c1c534cea
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: csharp_linux
-          bazel: test //csharp/... --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=/home/bazel
+          bazel: test //csharp:conformance_test --action_env=DOTNET_CLI_TELEMETRY_OPTOUT=1 --test_env=DOTNET_CLI_HOME=/home/bazel
 
   windows:
     name: Windows
@@ -42,11 +60,22 @@ jobs:
         with:
           dotnet-version: '6.0.x'
 
-      - name: Build
-        run: dotnet build csharp/src/Google.Protobuf.sln
+      # Workaround for incompatibility between gcloud and windows-2019 runners.
+      - name: Install Python
+        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+        with:
+          python-version: '3.9'
+      - name: Use custom python for gcloud
+        run: echo "CLOUDSDK_PYTHON=${Python3_ROOT_DIR}\\python3" >> $GITHUB_ENV
+        shell: bash
 
-      - name: Run Tests
-        run: dotnet test csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj
+      - name: Run tests
+        uses: protocolbuffers/protobuf-ci/bash@v2
+        with:
+          credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
+          command: |
+            dotnet build csharp/src/Google.Protobuf.sln
+            dotnet test csharp/src/Google.Protobuf.Test/Google.Protobuf.Test.csproj
 
   linux-aarch64:
     name: Linux aarch64

diff --git a/.github/workflows/test_php.yml b/.github/workflows/test_php.yml
@@ -171,7 +171,7 @@ jobs:
         run: brew install coreutils gd
 
       - name: Pin PHP version
-        uses: shivammathur/setup-php@7fdd3ece872ec7ec4c098ae5ab7637d5e0a96067 # 2.26.0
+        uses: shivammathur/setup-php@a4e22b60bbb9c1021113f2860347b0759f66fe5d # 2.30.0
         with:
           php-version: ${{ matrix.version }}
 

diff --git a/.github/workflows/test_ruby.yml b/.github/workflows/test_ruby.yml
@@ -19,10 +19,9 @@ jobs:
         include:
           # Test both FFI and Native implementations on the highest and lowest
           # Ruby versions for CRuby and JRuby, but only on Bazel 5.x.
-          - { name: Ruby 2.7, ruby: ruby-2.7.0, ffi: NATIVE }
+          - { name: Ruby 3.0, ruby: ruby-3.0.2, ffi: NATIVE }
           # TODO Re-enable these once flakes are fixed
-          #- { name: Ruby 2.7, ruby: ruby-2.7.0, ffi: FFI }
-          - { name: Ruby 3.0, ruby: ruby-3.0.2 }
+          #- { name: Ruby 3.0, ruby: ruby-3.0.2, ffi: FFI }
           - { name: Ruby 3.1, ruby: ruby-3.1.0 }
           - { name: Ruby 3.2, ruby: ruby-3.2.0 }
           - { name: Ruby 3.3, ruby: ruby-3.3.0, ffi: NATIVE }
@@ -66,12 +65,11 @@ jobs:
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/docker@v2
         with:
-          image: i386/ruby:2.7.3-buster
+          image: i386/ruby:3.0.2-buster
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          # Pin to Ruby 2.7 compatible bundler version.
           command: >-
             /bin/bash -cex '
-            gem install bundler -v 2.4.22;
+            gem install bundler -v 2.5.6;
             cd /workspace/ruby;
             bundle;
             PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} rake;
@@ -98,12 +96,11 @@ jobs:
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/docker@v2
         with:
-          image: arm64v8/ruby:2.7.3-buster
+          image: arm64v8/ruby:3.0.2-buster
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
-          # Pin to Ruby 2.7 compatible bundler version.
           command: >-
             /bin/bash -cex '
-            gem install bundler -v 2.4.22;
+            gem install bundler -v 2.5.6;
             cd /workspace/ruby;
             bundle;
             PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }} rake;
@@ -118,10 +115,9 @@ jobs:
         # Test both FFI and Native implementations on the highest and lowest
         # Ruby versions for CRuby, but only on Bazel 5.x.
         # Quote versions numbers otherwise 3.0 will render as 3
-        - { version: "2.7", ffi: NATIVE }
+        - { version: "3.0", ffi: NATIVE }
         # TODO Re-enable these once flakes are fixed
-        #- { version: "2.7", ffi: FFI }
-        - { version: "3.0" }
+        #- { version: "3.0", ffi: FFI }
         - { version: "3.1" }
         - { version: "3.2" }
         - { version: "3.3", ffi: NATIVE }
@@ -158,9 +154,8 @@ jobs:
         include:
           # Test both FFI and Native implementations on the highest and lowest
           # Ruby versions for CRuby and JRuby, but only on Bazel 5.x.
-          - { name: Ruby 2.7, ruby: ruby-2.7.0, ffi: NATIVE }
-          - { name: Ruby 2.7, ruby: ruby-2.7.0, ffi: FFI }
-          - { name: Ruby 3.0, ruby: ruby-3.0.2}
+          - { name: Ruby 3.0, ruby: ruby-3.0.2, ffi: NATIVE}
+          - { name: Ruby 3.0, ruby: ruby-3.0.2, ffi: FFI}
           - { name: Ruby 3.1, ruby: ruby-3.1.0}
           - { name: Ruby 3.2, ruby: ruby-3.2.0}
           - { name: Ruby 3.3, ruby: ruby-3.3.0, ffi: NATIVE }

diff --git a/README.md b/README.md
@@ -16,6 +16,23 @@ This README file contains protobuf installation instructions. To install
 protobuf, you need to install the protocol compiler (used to compile .proto
 files) and the protobuf runtime for your chosen programming language.
 
+Working With Protobuf Source Code
+---------------------------------
+
+Most users will find working from
+[supported releases](https://github.com/protocolbuffers/protobuf/releases) to be
+the easiest path.
+
+If you choose to work from the head revision of the main branch your build will
+occasionally be broken by source-incompatible changes and insufficiently-tested
+(and therefore broken) behavior.
+
+If you are using C++ or otherwise need to build protobuf from source as a part
+of your project, you should pin to a release commit on a release branch.
+
+This is because even release branches can experience some instability in between
+release commits.
+
 Protobuf Compiler Installation
 ------------------------------
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -82,6 +82,14 @@ load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependenci
 
 apple_rules_dependencies()
 
+load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies")
+
+apple_support_dependencies()
+
+load("@rules_cc//cc:repositories.bzl", "rules_cc_dependencies")
+
+rules_cc_dependencies()
+
 # For `kt_jvm_library`
 load("@io_bazel_rules_kotlin//kotlin:repositories.bzl", "kotlin_repositories")
 

diff --git a/benchmarks/benchmark.cc b/benchmarks/benchmark.cc
@@ -214,7 +214,7 @@ static void BM_LoadAdsDescriptor_Proto2(benchmark::State& state) {
     for (auto file : serialized_files) {
       absl::string_view input(file.data, file.size);
       auto proto =
-          protobuf::Arena::CreateMessage<protobuf::FileDescriptorProto>(&arena);
+          protobuf::Arena::Create<protobuf::FileDescriptorProto>(&arena);
       bool ok = proto->ParseFrom<protobuf::MessageLite::kMergePartial>(input) &&
                 pool.BuildFile(*proto) != nullptr;
       if (!ok) {
@@ -292,7 +292,7 @@ struct Proto2Factory<NoArena, P> {
 template <class P>
 struct Proto2Factory<UseArena, P> {
  public:
-  P* GetProto() { return protobuf::Arena::CreateMessage<P>(&arena); }
+  P* GetProto() { return protobuf::Arena::Create<P>(&arena); }
 
  private:
   protobuf::Arena arena;
@@ -302,7 +302,7 @@ template <class P>
 struct Proto2Factory<InitBlock, P> {
  public:
   Proto2Factory() : arena(GetOptions()) {}
-  P* GetProto() { return protobuf::Arena::CreateMessage<P>(&arena); }
+  P* GetProto() { return protobuf::Arena::Create<P>(&arena); }
 
  private:
   protobuf::ArenaOptions GetOptions() {

diff --git a/ci/macOS.bazelrc b/ci/macOS.bazelrc
@@ -1,3 +1,4 @@
 import common.bazelrc
 
 build --cxxopt=-std=c++14 --host_cxxopt=-std=c++14
+common --repo_env=BAZEL_NO_APPLE_CPP_TOOLCHAIN=1
diff --git a/conformance/conformance_rust.rs b/conformance/conformance_rust.rs
@@ -62,13 +62,13 @@ fn do_test(req: &ConformanceRequest) -> ConformanceResponse {
     let message_type = req.message_type();
 
     if req.requested_output_format() != WireFormat::Protobuf {
-        resp.skipped_mut().set("only wire format output implemented");
+        resp.set_skipped("only wire format output implemented");
         return resp;
     }
 
     let bytes = match req.protobuf_payload_opt() {
         Unset(_) => {
-            resp.skipped_mut().set("only wire format input implemented");
+            resp.set_skipped("only wire format input implemented");
             return resp;
         }
         Set(bytes) => bytes,
@@ -78,39 +78,39 @@ fn do_test(req: &ConformanceRequest) -> ConformanceResponse {
         b"protobuf_test_messages.proto2.TestAllTypesProto2" => {
             let mut proto = TestAllTypesProto2::new();
             if let Err(_) = proto.deserialize(bytes) {
-                resp.parse_error_mut().set("failed to parse bytes");
+                resp.set_parse_error("failed to parse bytes");
                 return resp;
             }
             proto.serialize()
         }
         b"protobuf_test_messages.proto3.TestAllTypesProto3" => {
             let mut proto = TestAllTypesProto3::new();
             if let Err(_) = proto.deserialize(bytes) {
-                resp.parse_error_mut().set("failed to parse bytes");
+                resp.set_parse_error("failed to parse bytes");
                 return resp;
             }
             proto.serialize()
         }
         b"protobuf_test_messages.editions.proto2.TestAllTypesProto2" => {
             let mut proto = EditionsTestAllTypesProto2::new();
             if let Err(_) = proto.deserialize(bytes) {
-                resp.parse_error_mut().set("failed to parse bytes");
+                resp.set_parse_error("failed to parse bytes");
                 return resp;
             }
             proto.serialize()
         }
         b"protobuf_test_messages.editions.proto3.TestAllTypesProto3" => {
             let mut proto = EditionsTestAllTypesProto3::new();
             if let Err(_) = proto.deserialize(bytes) {
-                resp.parse_error_mut().set("failed to parse bytes");
+                resp.set_parse_error("failed to parse bytes");
                 return resp;
             }
             proto.serialize()
         }
         _ => panic!("unexpected msg type {message_type}"),
     };
 
-    resp.protobuf_payload_mut().set(serialized);
+    resp.set_protobuf_payload(serialized);
     return resp;
 }
 

diff --git a/csharp/generate_protos.sh b/csharp/generate_protos.sh
@@ -73,6 +73,8 @@ $PROTOC -Isrc -I. \
     src/google/protobuf/unittest_well_known_types.proto \
     src/google/protobuf/test_messages_proto3.proto \
     src/google/protobuf/test_messages_proto2.proto \
+    src/google/protobuf/unittest_features.proto \
+    src/google/protobuf/unittest_legacy_features.proto \
     src/google/protobuf/unittest_proto3_optional.proto \
     src/google/protobuf/unittest_retention.proto