blaze: add 'json' module to all Starlark environments

This change predeclares 'json', the new Starlark module
for JSON encoding/decoding/indenting, in all Bazel Starlark
environments (alongside depset, select, etc).

The new function works for any legal value, not just struct,
and avoids polluting the struct field namespace.

struct.to_json is deprecated, along with struct.to_proto,
and will be disabled by the new flag --incompatible_struct_has_no_methods.
(The replacement for to_proto will be added shortly.)

Updates bazelbuild/starlark#83
Updates #3732
Updates #1813

RELNOTES: The Starlark json module is now available.
Use json.encode(x) to encode a Starlark value as JSON.
struct.to_json(x) is deprecated and will be disabled by
the --incompatible_struct_has_no_methods flag.
PiperOrigin-RevId: 338259618

Author: adonovan

src/main/java/com/google/devtools/build/docgen/StarlarkDocumentationProcessor.java

@@ -236,6 +236,7 @@ static Category of(StarlarkBuiltin annot) {
         case DocCategory.NONE:
           return NONE;
         case "core": // interpreter built-ins (e.g. int)
+        case "core.lib": // Starlark standard modules (e.g. json)
           return CORE;
         case "": // no annotation
           return TOP_LEVEL_TYPE;

src/main/java/com/google/devtools/build/lib/packages/BUILD

@@ -44,7 +44,6 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/starlarkbuildapi/core",
         "//src/main/java/com/google/devtools/build/lib/util",
         "//src/main/java/com/google/devtools/build/lib/util:detailed_exit_code",
-        "//src/main/java/com/google/devtools/build/lib/util:exit_code",
         "//src/main/java/com/google/devtools/build/lib/util:filetype",
         "//src/main/java/com/google/devtools/build/lib/util:string",
         "//src/main/java/com/google/devtools/build/lib/vfs",
@@ -53,6 +52,7 @@ java_library(
         "//src/main/java/com/google/devtools/common/options",
         "//src/main/java/net/starlark/java/annot",
         "//src/main/java/net/starlark/java/eval",
+        "//src/main/java/net/starlark/java/lib/json",
         "//src/main/java/net/starlark/java/spelling",
         "//src/main/java/net/starlark/java/syntax",
         "//src/main/protobuf:build_java_proto",

src/main/java/com/google/devtools/build/lib/packages/StarlarkLibrary.java

@@ -36,6 +36,7 @@
 import net.starlark.java.eval.Sequence;
 import net.starlark.java.eval.Starlark;
 import net.starlark.java.eval.StarlarkThread;
+import net.starlark.java.lib.json.Json;
 import net.starlark.java.syntax.Location;
 
 /**
@@ -51,15 +52,16 @@ public final class StarlarkLibrary {
   private StarlarkLibrary() {} // uninstantiable
 
   /**
-   * A library of Starlark functions (keyed by name) that are not part of core Starlark but are
-   * common to all Bazel Starlark file environments (BUILD, .bzl, and WORKSPACE). Examples: depset,
-   * select.
+   * A library of Starlark values (keyed by name) that are not part of core Starlark but are common
+   * to all Bazel Starlark file environments (BUILD, .bzl, and WORKSPACE). Examples: depset, select,
+   * json.
    */
   public static final ImmutableMap<String, Object> COMMON = initCommon();
 
   private static ImmutableMap<String, Object> initCommon() {
     ImmutableMap.Builder<String, Object> env = ImmutableMap.builder();
     Starlark.addMethods(env, new CommonLibrary());
+    Starlark.addModule(env, Json.INSTANCE);
     return env.build();
   }
 

src/main/java/com/google/devtools/build/lib/packages/semantics/BuildLanguageOptions.java

@@ -257,6 +257,21 @@ public class BuildLanguageOptions extends OptionsBase implements Serializable {
               + " https://github.com/bazelbuild/bazel/issues/8830 for details.")
   public boolean experimentalAllowTagsPropagation;
 
+  @Option(
+      name = "incompatible_struct_has_no_methods",
+      defaultValue = "false",
+      documentationCategory = OptionDocumentationCategory.STARLARK_SEMANTICS,
+      effectTags = {OptionEffectTag.BUILD_FILE_SEMANTICS},
+      metadataTags = {
+        OptionMetadataTag.INCOMPATIBLE_CHANGE,
+        OptionMetadataTag.TRIGGERED_BY_ALL_INCOMPATIBLE_CHANGES
+      },
+      help =
+          "Disables the to_json and to_proto methods of struct, which pollute the struct field"
+              + " namespace. Instead, use json.encode or json.encode_indent for JSON, or"
+              + " proto.encode_text for textproto.")
+  public boolean incompatibleStructHasNoMethods;
+
   @Option(
       name = "incompatible_always_check_depset_elements",
       defaultValue = "true",
@@ -637,6 +652,7 @@ public StarlarkSemantics toStarlarkSemantics() {
             .setBool(INCOMPATIBLE_RUN_SHELL_COMMAND_STRING, incompatibleRunShellCommandString)
             .setBool(
                 StarlarkSemantics.INCOMPATIBLE_STRING_REPLACE_COUNT, incompatibleStringReplaceCount)
+            .setBool(INCOMPATIBLE_STRUCT_HAS_NO_METHODS, incompatibleStructHasNoMethods)
             .setBool(
                 INCOMPATIBLE_VISIBILITY_PRIVATE_ATTRIBUTES_AT_DEFINITION,
                 incompatibleVisibilityPrivateAttributesAtDefinition)
@@ -720,6 +736,8 @@ public StarlarkSemantics toStarlarkSemantics() {
       "-incompatible_restrict_string_escapes";
   public static final String INCOMPATIBLE_RUN_SHELL_COMMAND_STRING =
       "-incompatible_run_shell_command_string";
+  public static final String INCOMPATIBLE_STRUCT_HAS_NO_METHODS =
+      "-incompatible_struct_has_no_methods";
   public static final String INCOMPATIBLE_USE_CC_CONFIGURE_FROM_RULES_CC =
       "-incompatible_use_cc_configure_from_rules";
   public static final String INCOMPATIBLE_VISIBILITY_PRIVATE_ATTRIBUTES_AT_DEFINITION =

src/main/java/com/google/devtools/build/lib/starlarkbuildapi/core/BUILD

@@ -25,6 +25,7 @@ java_library(
     srcs = glob(["*.java"]),
     deps = [
         "//src/main/java/com/google/devtools/build/docgen/annot",
+        "//src/main/java/com/google/devtools/build/lib/packages/semantics",
         "//src/main/java/net/starlark/java/annot",
         "//src/main/java/net/starlark/java/eval",
         "//third_party:guava",

src/main/java/com/google/devtools/build/lib/starlarkbuildapi/core/StructApi.java

@@ -16,6 +16,7 @@
 
 import com.google.devtools.build.docgen.annot.DocCategory;
 import com.google.devtools.build.docgen.annot.StarlarkConstructor;
+import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
 import net.starlark.java.annot.Param;
 import net.starlark.java.annot.StarlarkBuiltin;
 import net.starlark.java.annot.StarlarkMethod;
@@ -62,7 +63,9 @@ public interface StructApi extends StarlarkValue {
               + "#   key: 2\n"
               + "#   value: 1\n"
               + "# }\n"
-              + "</pre>")
+              + "</pre>",
+      // TODO(adonovan): CL 338119348 defines proto.encode_text(x) and deprecates this function.
+      disableWithFlag = BuildLanguageOptions.INCOMPATIBLE_STRUCT_HAS_NO_METHODS)
   String toProto() throws EvalException;
 
   @StarlarkMethod(
@@ -82,7 +85,10 @@ public interface StructApi extends StarlarkValue {
               + "struct(key=[struct(inner_key=1), struct(inner_key=2)]).to_json()\n"
               + "# {\"key\":[{\"inner_key\":1},{\"inner_key\":2}]}\n\n"
               + "struct(key=struct(inner_key=struct(inner_inner_key='text'))).to_json()\n"
-              + "# {\"key\":{\"inner_key\":{\"inner_inner_key\":\"text\"}}}\n</pre>")
+              + "# {\"key\":{\"inner_key\":{\"inner_inner_key\":\"text\"}}}\n</pre>."
+              + "<p>Deprecated: instead, use json.encode(x) or json.encode_indent(x), which work"
+              + " for values other than structs and do not pollute the struct field namespace. ",
+      disableWithFlag = BuildLanguageOptions.INCOMPATIBLE_STRUCT_HAS_NO_METHODS)
   String toJson() throws EvalException;
 
   /** Callable Provider for new struct objects. */

src/test/java/com/google/devtools/build/lib/packages/semantics/ConsistencyTest.java

@@ -151,6 +151,7 @@ private static BuildLanguageOptions buildRandomOptions(Random rand) throws Excep
         "--incompatible_objc_provider_remove_compile_info=" + rand.nextBoolean(),
         "--incompatible_run_shell_command_string=" + rand.nextBoolean(),
         "--incompatible_string_replace_count=" + rand.nextBoolean(),
+        "--incompatible_struct_has_no_methods=" + rand.nextBoolean(),
         "--incompatible_visibility_private_attributes_at_definition=" + rand.nextBoolean(),
         "--incompatible_require_linker_input_cc_api=" + rand.nextBoolean(),
         "--incompatible_restrict_string_escapes=" + rand.nextBoolean(),
@@ -207,6 +208,7 @@ private static StarlarkSemantics buildRandomSemantics(Random rand) {
             BuildLanguageOptions.INCOMPATIBLE_OBJC_PROVIDER_REMOVE_COMPILE_INFO, rand.nextBoolean())
         .setBool(BuildLanguageOptions.INCOMPATIBLE_RUN_SHELL_COMMAND_STRING, rand.nextBoolean())
         .setBool(StarlarkSemantics.INCOMPATIBLE_STRING_REPLACE_COUNT, rand.nextBoolean())
+        .setBool(BuildLanguageOptions.INCOMPATIBLE_STRUCT_HAS_NO_METHODS, rand.nextBoolean())
         .setBool(
             BuildLanguageOptions.INCOMPATIBLE_VISIBILITY_PRIVATE_ATTRIBUTES_AT_DEFINITION,
             rand.nextBoolean())

src/test/java/com/google/devtools/build/lib/starlark/StarlarkRuleClassFunctionsTest.java

@@ -1080,6 +1080,20 @@ private void checkJson(String from, String expected) throws Exception {
     assertThat(result).isEqualTo(expected);
   }
 
+  @Test
+  public void testStarlarkJsonModule() throws Exception {
+    // struct.to_json is deprecated.
+    // java.starlark.net's json module is its replacement.
+    setBuildLanguageOptions("--incompatible_struct_has_no_methods=false");
+    checkJson("json.encode(struct(name=True))", "{\"name\":true}");
+    checkJson("json.encode([1, 2])", "[1,2]"); // works for non-structs too
+    checkJson("str(dir(struct()))", "[\"to_json\", \"to_proto\"]");
+
+    setBuildLanguageOptions("--incompatible_struct_has_no_methods=true");
+    ev.checkEvalErrorContains("no field or method 'to_json'", "struct(name=True).to_json()");
+    checkJson("str(dir(struct()))", "[]"); // no to_{json,proto}
+  }
+
   @Test
   public void testJsonBooleanFields() throws Exception {
     checkJson("struct(name=True).to_json()", "{\"name\":true}");