Move upb minitable code generator into protoc

The main goal of this is to incorporate the minitable generator into our
released protoc binaries so that Rust users don't have to build it from source.

PiperOrigin-RevId: 728785507

Author: Adam Cozzette

.github/workflows/test_cpp.yml

@@ -135,7 +135,7 @@ jobs:
             -c "set -ex;
             sccache -z;
             cmake . -DWITH_PROTOC=/workspace/${{ steps.cross-compile.outputs.protoc }}
-            -Dprotobuf_BUILD_LIBUPB=OFF -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=17
+            -Dprotobuf_BUILD_CONFORMANCE=ON -DCMAKE_CXX_STANDARD=17
             -Dprotobuf_WITH_ZLIB=OFF ${{ env.SCCACHE_CMAKE_FLAGS }};
             cmake --build . --parallel 20;
             ctest --parallel 20;

.github/workflows/test_upb.yml

@@ -74,7 +74,7 @@ jobs:
           bazel-cache: "upb-bazel-gcc"
           bazel: >-
             test --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt
-            --copt="-Wno-error=maybe-uninitialized" --copt="-Wno-error=attributes"
+            --copt="-Wno-error=maybe-uninitialized"
             //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
 
   windows:

cmake/libprotoc.cmake

@@ -17,7 +17,7 @@ if(protobuf_HAVE_LD_VERSION_SCRIPT)
     LINK_DEPENDS ${protobuf_SOURCE_DIR}/src/libprotoc.map)
 endif()
 target_link_libraries(libprotoc PRIVATE libprotobuf)
-target_link_libraries(libprotoc PUBLIC ${protobuf_ABSL_USED_TARGETS})
+target_link_libraries(libprotoc PUBLIC libupb ${protobuf_ABSL_USED_TARGETS})
 protobuf_configure_target(libprotoc)
 if(protobuf_BUILD_SHARED_LIBS)
   target_compile_definitions(libprotoc

pkg/BUILD.bazel

@@ -194,6 +194,7 @@ cc_dist_library(
     dist_deps = [
         ":protobuf",
         ":protobuf_lite",
+        ":upb",
     ],
     tags = ["manual"],
     deps = [
@@ -207,6 +208,7 @@ cc_dist_library(
         "//src/google/protobuf/compiler/python",
         "//src/google/protobuf/compiler/ruby",
         "//src/google/protobuf/compiler/rust",
+        "//upb_generator/minitable:generator",
     ],
 )
 

src/google/protobuf/compiler/BUILD.bazel

@@ -238,6 +238,7 @@ cc_library(
         "//src/google/protobuf/compiler/python",
         "//src/google/protobuf/compiler/ruby",
         "//src/google/protobuf/compiler/rust",
+        "//upb_generator/minitable:generator_stage0",
         "@abseil-cpp//absl/log:initialize",
     ],
 )

src/google/protobuf/compiler/main.cc

@@ -17,6 +17,7 @@
 #include "google/protobuf/compiler/python/pyi_generator.h"
 #include "google/protobuf/compiler/ruby/ruby_generator.h"
 #include "google/protobuf/compiler/rust/generator.h"
+#include "upb_generator/minitable/generator.h"
 
 #ifdef DISABLE_PROTOC_CONFIG
 #include "google/protobuf/compiler/allowlists/allowlist.h"
@@ -106,6 +107,11 @@ int ProtobufMain(int argc, char* argv[]) {
   rust::RustGenerator rust_generator;
   cli.RegisterGenerator("--rust_out", "--rust_opt", &rust_generator,
                         "Generate Rust sources.");
+
+  // upb minitables
+  upb::generator::MiniTableGenerator minitable_generator;
+  cli.RegisterGenerator("--upb_minitable_out", "--upb_minitable_opt",
+                        &minitable_generator, "Generate upb minitables");
 #ifdef DISABLE_PROTOC_CONFIG
   auto cleanup = internal::DisableAllowlistInternalOnly();
 #endif  // DISABLE_PROTOC_CONFIG

upb/bazel/build_defs.bzl

@@ -11,6 +11,9 @@ _DEFAULT_CPPOPTS = []
 _DEFAULT_COPTS = [
     # this is a compile error in C++ clang and GNU C, but not clang C by default
     "-Werror=incompatible-pointer-types",
+    # GCC does not seem to suppport the no_sanitize attribute in some places
+    # where we use it.
+    "-Wno-error=attributes",
 ]
 
 _DEFAULT_CPPOPTS.extend([

upb_generator/bootstrap_compiler.bzl

@@ -20,6 +20,7 @@ _extra_proto_path = "-I$$(dirname $(location @com_google_protobuf//:descriptor_p
 
 # This visibility is used automatically for anything used by the bootstrapping process.
 _bootstrap_visibility = [
+    "//src/google/protobuf/compiler:__pkg__",
     "//third_party/upb/github:__pkg__",
     "//upb_generator:__subpackages__",
     "//upb/reflection:__pkg__",

upb_generator/minitable/BUILD

@@ -30,13 +30,17 @@ bootstrap_cc_library(
         "//upb_generator:file_layout",
         "//upb/reflection:descriptor_upb_proto",
         "//upb/reflection:reflection",
+        "//upb_generator:plugin",
     ],
     copts = UPB_DEFAULT_CPPOPTS,
+    visibility = ["//pkg:__pkg__"],
     deps = [
         ":names",
         ":names_internal",
+        "//src/google/protobuf:port",
         "//src/google/protobuf/compiler:code_generator",
         "//src/google/protobuf/compiler:code_generator_lite",
+        "//src/google/protobuf/compiler:plugin",
         "//upb:base",
         "//upb:mem",
         "//upb:mini_table",

upb_generator/minitable/generator.cc

@@ -21,6 +21,8 @@
 #include "absl/strings/string_view.h"
 #include "absl/strings/substitute.h"
 #include "google/protobuf/compiler/code_generator.h"
+#include "google/protobuf/compiler/code_generator_lite.h"
+#include "upb/mem/arena.hpp"
 #include "upb/mini_table/enum.h"
 #include "upb/mini_table/field.h"
 #include "upb/mini_table/internal/field.h"
@@ -32,6 +34,7 @@
 #include "upb_generator/minitable/fasttable.h"
 #include "upb_generator/minitable/names.h"
 #include "upb_generator/minitable/names_internal.h"
+#include "upb_generator/plugin.h"
 
 // Must be last.
 #include "upb/port/def.inc"
@@ -468,5 +471,72 @@ void WriteMiniTableMultipleSources(
   }
 }
 
+std::string SourceFilename(upb::FileDefPtr file) {
+  return StripExtension(file.name()) + ".upb_minitable.c";
+}
+
+void GenerateFile(const DefPoolPair& pools, upb::FileDefPtr file,
+                  const MiniTableOptions& options,
+                  google::protobuf::compiler::GeneratorContext* context) {
+  Output h_output;
+  WriteMiniTableHeader(pools, file, options, h_output);
+  {
+    auto stream = absl::WrapUnique(
+        context->Open(MiniTableHeaderFilename(file.name(), false)));
+    ABSL_CHECK(stream->WriteCord(absl::Cord(h_output.output())));
+  }
+
+  Output c_output;
+  WriteMiniTableSource(pools, file, options, c_output);
+  {
+    auto stream = absl::WrapUnique(context->Open(SourceFilename(file)));
+    ABSL_CHECK(stream->WriteCord(absl::Cord(c_output.output())));
+  }
+
+  if (options.one_output_per_message) {
+    WriteMiniTableMultipleSources(pools, file, options, context);
+  }
+}
+
+bool ParseOptions(MiniTableOptions* options, absl::string_view parameter,
+                  std::string* error) {
+  for (const auto& pair : ParseGeneratorParameter(parameter)) {
+    if (pair.first == "bootstrap_stage") {
+      options->bootstrap = true;
+    } else if (pair.first == "experimental_strip_nonfunctional_codegen") {
+      options->strip_nonfunctional_codegen = true;
+    } else if (pair.first == "one_output_per_message") {
+      options->one_output_per_message = true;
+    } else {
+      *error = absl::Substitute("Unknown parameter: $0", pair.first);
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool MiniTableGenerator::GenerateAll(
+    const std::vector<const google::protobuf::FileDescriptor*>& files,
+    const std::string& parameter,
+    google::protobuf::compiler::GeneratorContext* generator_context,
+    std::string* error) const {
+  MiniTableOptions options;
+  if (!ParseOptions(&options, parameter, error)) {
+    return false;
+  }
+
+  upb::Arena arena;
+  DefPoolPair pools;
+  absl::flat_hash_set<std::string> files_seen;
+  for (const auto* file : files) {
+    PopulateDefPool(file, &arena, &pools, &files_seen);
+    upb::FileDefPtr upb_file = pools.GetFile(file->name());
+    GenerateFile(pools, upb_file, options, generator_context);
+  }
+
+  return true;
+}
+
 }  // namespace generator
 }  // namespace upb