Mirrors
/
protobuf
mirror of https://github.com/protocolbuffers/protobuf.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement new Debug API with redaction. 

Implement emittingSingleLine TextFormat printer option.

PiperOrigin-RevId: 612925460
Protobuf Team Bot 2024-03-05 12:08:36 -08:00 committed by Copybara-Service
parent ec126a5069
commit f5e4484ec4
5 changed files with 383 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
java/core/BUILD.bazel
View File

@ -520,6 +520,7 @@ LITE_TEST_EXCLUSIONS = [
"src/test/java/com/google/protobuf/AnyTest.java",
"src/test/java/com/google/protobuf/CodedInputStreamTest.java",
"src/test/java/com/google/protobuf/DeprecatedFieldTest.java",
"src/test/java/com/google/protobuf/DebugFormatTest.java",
"src/test/java/com/google/protobuf/DescriptorsTest.java",
"src/test/java/com/google/protobuf/DiscardUnknownFieldsTest.java",
"src/test/java/com/google/protobuf/DynamicMessageTest.java",

83
java/core/src/main/java/com/google/protobuf/DebugFormat.java Normal file
View File

@ -0,0 +1,83 @@
package com.google.protobuf;
import com.google.protobuf.Descriptors.FieldDescriptor;
/**
* Provides an explicit API for unstable, redacting debug output suitable for debug logging. This
* implementation is based on TextFormat, but should not be parsed.
*/
final class DebugFormat {
private final boolean isSingleLine;
private DebugFormat(boolean singleLine) {
isSingleLine = singleLine;
}
public static DebugFormat singleLine() {
return new DebugFormat(true);
}
public static DebugFormat multiline() {
return new DebugFormat(false);
}
public String toString(MessageOrBuilder message) {
return TextFormat.printer()
.emittingSingleLine(this.isSingleLine)
.enablingSafeDebugFormat(true)
.printToString(message);
}
public String toString(FieldDescriptor field, Object value) {
return TextFormat.printer()
.emittingSingleLine(this.isSingleLine)
.enablingSafeDebugFormat(true)
.printFieldToString(field, value);
}
public String toString(UnknownFieldSet fields) {
return TextFormat.printer()
.emittingSingleLine(this.isSingleLine)
.enablingSafeDebugFormat(true)
.printToString(fields);
}
public Object lazyToString(MessageOrBuilder message) {
return new LazyDebugOutput(message, this.isSingleLine);
}
public Object lazyToString(UnknownFieldSet fields) {
return new LazyDebugOutput(fields, this.isSingleLine);
}
private static class LazyDebugOutput {
private final MessageOrBuilder message;
private final UnknownFieldSet fields;
private final boolean isSingleLine;
LazyDebugOutput(MessageOrBuilder message, boolean isSingleLine) {
this.message = message;
this.fields = null;
this.isSingleLine = isSingleLine;
}
LazyDebugOutput(UnknownFieldSet fields, boolean isSingleLine) {
this.fields = fields;
this.message = null;
this.isSingleLine = isSingleLine;
}
@Override
public String toString() {
if (message != null) {
return this.isSingleLine
? DebugFormat.singleLine().toString(message)
: DebugFormat.multiline().toString(message);
}
return this.isSingleLine
? DebugFormat.singleLine().toString(fields)
: DebugFormat.multiline().toString(fields);
}
}
}

148
java/core/src/main/java/com/google/protobuf/TextFormat.java
View File

@ -38,6 +38,13 @@ public final class TextFormat {
private static final String DEBUG_STRING_SILENT_MARKER = "\t ";
private static final String REDACTED_MARKER = getRedactedMarker();
private static String getRedactedMarker() {
String redactedMarker = "[REDACTED]";
return redactedMarker;
}
/**
* Generates a human readable form of this message, useful for debugging and other purposes, with
* no newline characters. This is just a trivial wrapper around {@link
@ -58,7 +65,7 @@ public final class TextFormat {
*/
public static void printUnknownFieldValue(
final int tag, final Object value, final Appendable output) throws IOException {
printUnknownFieldValue(tag, value, multiLineOutput(output));
printUnknownFieldValue(tag, value, setLineOutput(output, false));
}
private static void printUnknownFieldValue(
@ -109,7 +116,11 @@ public final class TextFormat {
// Printer instance which escapes non-ASCII characters.
private static final Printer DEFAULT =
new Printer(
true, TypeRegistry.getEmptyTypeRegistry(), ExtensionRegistryLite.getEmptyRegistry());
true,
TypeRegistry.getEmptyTypeRegistry(),
ExtensionRegistryLite.getEmptyRegistry(),
false,
false);
/** Whether to escape non ASCII characters with backslash and octal. */
private final boolean escapeNonAscii;
@ -117,13 +128,22 @@ public final class TextFormat {
private final TypeRegistry typeRegistry;
private final ExtensionRegistryLite extensionRegistry;
/** Whether to enable redaction of sensitive fields and unstable/undeserializable output. */
private final boolean enablingSafeDebugFormat;
private final boolean singleLine;
private Printer(
boolean escapeNonAscii,
TypeRegistry typeRegistry,
ExtensionRegistryLite extensionRegistry) {
ExtensionRegistryLite extensionRegistry,
boolean enablingSafeDebugFormat,
boolean singleLine) {
this.escapeNonAscii = escapeNonAscii;
this.typeRegistry = typeRegistry;
this.extensionRegistry = extensionRegistry;
this.enablingSafeDebugFormat = enablingSafeDebugFormat;
this.singleLine = singleLine;
}
/**
@ -136,7 +156,8 @@ public final class TextFormat {
* with the escape mode set to the given parameter.
*/
public Printer escapingNonAscii(boolean escapeNonAscii) {
return new Printer(escapeNonAscii, typeRegistry, extensionRegistry);
return new Printer(
escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine);
}
/**
@ -149,7 +170,8 @@ public final class TextFormat {
if (this.typeRegistry != TypeRegistry.getEmptyTypeRegistry()) {
throw new IllegalArgumentException("Only one typeRegistry is allowed.");
}
return new Printer(escapeNonAscii, typeRegistry, extensionRegistry);
return new Printer(
escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine);
}
/**
@ -162,7 +184,34 @@ public final class TextFormat {
if (this.extensionRegistry != ExtensionRegistryLite.getEmptyRegistry()) {
throw new IllegalArgumentException("Only one extensionRegistry is allowed.");
}
return new Printer(escapeNonAscii, typeRegistry, extensionRegistry);
return new Printer(
escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine);
}
/**
* Return a new Printer instance that outputs a redacted and unstable format suitable for
* debugging.
*
* @param enablingSafeDebugFormat If true, the new Printer will redact all proto fields that are
* marked by a debug_redact=true option, and apply an unstable prefix to the output.
* @return a new Printer that clones all other configurations from the current {@link Printer},
* with the enablingSafeDebugFormat mode set to the given parameter.
*/
Printer enablingSafeDebugFormat(boolean enablingSafeDebugFormat) {
return new Printer(
escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine);
}
/**
* Return a new Printer instance with the specified line formatting status.
*
* @param singleLine If true, the new Printer will output no newline characters.
* @return a new Printer that clones all other configurations from the current {@link Printer},
* with the singleLine mode set to the given parameter.
*/
Printer emittingSingleLine(boolean singleLine) {
return new Printer(
escapeNonAscii, typeRegistry, extensionRegistry, enablingSafeDebugFormat, singleLine);
}
/**
@ -171,12 +220,12 @@ public final class TextFormat {
* original Protocol Buffer system)
*/
public void print(final MessageOrBuilder message, final Appendable output) throws IOException {
print(message, multiLineOutput(output));
print(message, setLineOutput(output, this.singleLine));
}
/** Outputs a textual representation of {@code fields} to {@code output}. */
public void print(final UnknownFieldSet fields, final Appendable output) throws IOException {
printUnknownFields(fields, multiLineOutput(output));
printUnknownFields(fields, setLineOutput(output, this.singleLine));
}
private void print(final MessageOrBuilder message, final TextGenerator generator)
@ -188,6 +237,14 @@ public final class TextFormat {
printMessage(message, generator);
}
private void applyUnstablePrefix(final Appendable output) {
try {
output.append("");
} catch (IOException e) {
throw new IllegalStateException(e);
}
}
/**
* Attempt to print the 'google.protobuf.Any' message in a human-friendly format. Returns false
* if the message isn't a valid 'google.protobuf.Any' message (in which case the message should
@ -244,6 +301,9 @@ public final class TextFormat {
public String printFieldToString(final FieldDescriptor field, final Object value) {
try {
final StringBuilder text = new StringBuilder();
if (enablingSafeDebugFormat) {
applyUnstablePrefix(text);
}
printField(field, value, text);
return text.toString();
} catch (IOException e) {
@ -253,7 +313,7 @@ public final class TextFormat {
public void printField(final FieldDescriptor field, final Object value, final Appendable output)
throws IOException {
printField(field, value, multiLineOutput(output));
printField(field, value, setLineOutput(output, this.singleLine));
}
private void printField(
@ -358,12 +418,19 @@ public final class TextFormat {
public void printFieldValue(
final FieldDescriptor field, final Object value, final Appendable output)
throws IOException {
printFieldValue(field, value, multiLineOutput(output));
printFieldValue(field, value, setLineOutput(output, this.singleLine));
}
private void printFieldValue(
final FieldDescriptor field, final Object value, final TextGenerator generator)
throws IOException {
if (shouldRedact(field)) {
generator.print(REDACTED_MARKER);
if (field.getJavaType() == FieldDescriptor.JavaType.MESSAGE) {
generator.eol();
}
return;
}
switch (field.getType()) {
case INT32:
case SINT32:
@ -429,10 +496,54 @@ public final class TextFormat {
}
}
private boolean shouldRedactOptionValue(EnumValueDescriptor optionValue) {
if (optionValue.getOptions().hasDebugRedact()) {
return optionValue.getOptions().getDebugRedact();
}
return false;
}
// The criteria for redacting a field is as follows: 1) The enablingSafeDebugFormat printer
// option
// must be on. 2) The field must be marked by a debug_redact=true option, or is marked by an
// option with an enum value that is marked by a debug_redact=true option.
private boolean shouldRedact(final FieldDescriptor field) {
if (!this.enablingSafeDebugFormat) {
return false;
}
if (field.getOptions().hasDebugRedact()) {
return field.getOptions().getDebugRedact();
}
// Iterate through every option; if it's an enum, we check each enum value for debug_redact.
for (Map.Entry<Descriptors.FieldDescriptor, Object> entry :
field.getOptions().getAllFields().entrySet()) {
Descriptors.FieldDescriptor option = entry.getKey();
if (option.getType() != Descriptors.FieldDescriptor.Type.ENUM) {
continue;
}
if (option.isRepeated()) {
for (EnumValueDescriptor value : (List<EnumValueDescriptor>) entry.getValue()) {
if (shouldRedactOptionValue(value)) {
return true;
}
}
} else {
EnumValueDescriptor optionValue = (EnumValueDescriptor) entry.getValue();
if (shouldRedactOptionValue(optionValue)) {
return true;
}
}
}
return false;
}
/** Like {@code print()}, but writes directly to a {@code String} and returns it. */
public String printToString(final MessageOrBuilder message) {
try {
final StringBuilder text = new StringBuilder();
if (enablingSafeDebugFormat) {
applyUnstablePrefix(text);
}
print(message, text);
return text.toString();
} catch (IOException e) {
@ -443,6 +554,9 @@ public final class TextFormat {
public String printToString(final UnknownFieldSet fields) {
try {
final StringBuilder text = new StringBuilder();
if (enablingSafeDebugFormat) {
applyUnstablePrefix(text);
}
print(fields, text);
return text.toString();
} catch (IOException e) {
@ -457,7 +571,7 @@ public final class TextFormat {
public String shortDebugString(final MessageOrBuilder message) {
try {
final StringBuilder text = new StringBuilder();
print(message, singleLineOutput(text));
print(message, setLineOutput(text, true));
return text.toString();
} catch (IOException e) {
throw new IllegalStateException(e);
@ -471,7 +585,7 @@ public final class TextFormat {
public String shortDebugString(final FieldDescriptor field, final Object value) {
try {
final StringBuilder text = new StringBuilder();
printField(field, value, singleLineOutput(text));
printField(field, value, setLineOutput(text, true));
return text.toString();
} catch (IOException e) {
throw new IllegalStateException(e);
@ -485,7 +599,7 @@ public final class TextFormat {
public String shortDebugString(final UnknownFieldSet fields) {
try {
final StringBuilder text = new StringBuilder();
printUnknownFields(fields, singleLineOutput(text));
printUnknownFields(fields, setLineOutput(text, true));
return text.toString();
} catch (IOException e) {
throw new IllegalStateException(e);
@ -637,12 +751,8 @@ public final class TextFormat {
}
}
private static TextGenerator multiLineOutput(Appendable output) {
return new TextGenerator(output, false);
}
private static TextGenerator singleLineOutput(Appendable output) {
return new TextGenerator(output, true);
private static TextGenerator setLineOutput(Appendable output, boolean singleLine) {
return new TextGenerator(output, singleLine);
}
/** An inner class for writing text to the output stream. */

164
java/core/src/test/java/com/google/protobuf/DebugFormatTest.java Normal file
View File

@ -0,0 +1,164 @@
package com.google.protobuf;
import static com.google.common.truth.Truth.assertThat;
import static protobuf_unittest.UnittestProto.redactedExtension;
import com.google.protobuf.Descriptors.FieldDescriptor;
import protobuf_unittest.UnittestProto.RedactedFields;
import protobuf_unittest.UnittestProto.TestNestedMessageRedaction;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
@RunWith(JUnit4.class)
public final class DebugFormatTest {
private static final String REDACTED_REGEX = getRedactedRegex();
private static final String UNSTABLE_PREFIX_SINGLE_LINE = getUnstablePrefix(true);
private static final String UNSTABLE_PREFIX_MULTILINE = getUnstablePrefix(false);
private static String getRedactedRegex() {
String redactedRegex = "\\[REDACTED\\]";
return redactedRegex;
}
private static String getUnstablePrefix(boolean singleLine) {
String unstablePrefix = "";
return unstablePrefix;
}
@Test
public void multilineMessageFormat_returnsMultiline() {
RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build();
String result = DebugFormat.multiline().toString(message);
assertThat(result)
.matches(
String.format("%soptional_unredacted_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE));
}
@Test
public void singleLineMessageFormat_returnsSingleLine() {
RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build();
String result = DebugFormat.singleLine().toString(message);
assertThat(result)
.matches(
String.format("%soptional_unredacted_string: \"foo\"", UNSTABLE_PREFIX_SINGLE_LINE));
}
@Test
public void messageFormat_debugRedactFieldIsRedacted() {
RedactedFields message = RedactedFields.newBuilder().setOptionalRedactedString("foo").build();
String result = DebugFormat.multiline().toString(message);
assertThat(result)
.matches(
String.format(
"%soptional_redacted_string: %s\n", UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX));
}
@Test
public void messageFormat_debugRedactMessageIsRedacted() {
RedactedFields message =
RedactedFields.newBuilder()
.setOptionalRedactedMessage(
TestNestedMessageRedaction.newBuilder().setOptionalUnredactedNestedString("foo"))
.build();
String result = DebugFormat.multiline().toString(message);
assertThat(result)
.matches(
String.format(
"%soptional_redacted_message \\{\n %s\n\\}\n",
UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX));
}
@Test
public void messageFormat_debugRedactMapIsRedacted() {
RedactedFields message = RedactedFields.newBuilder().putMapRedactedString("foo", "bar").build();
String result = DebugFormat.multiline().toString(message);
assertThat(result)
.matches(
String.format(
"%smap_redacted_string \\{\\n %s\n\\}\n",
UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX));
}
@Test
public void messageFormat_debugRedactExtensionIsRedacted() {
RedactedFields message =
RedactedFields.newBuilder().setExtension(redactedExtension, "foo").build();
String result = DebugFormat.multiline().toString(message);
assertThat(result)
.matches(
String.format(
"%s\\[protobuf_unittest\\.redacted_extension\\]: %s\n",
UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX));
}
@Test
public void messageFormat_redactFalseIsNotRedacted() {
RedactedFields message =
RedactedFields.newBuilder().setOptionalRedactedFalseString("foo").build();
String result = DebugFormat.multiline().toString(message);
assertThat(result)
.matches(
String.format(
"%soptional_redacted_false_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE));
}
@Test
public void messageFormat_nonSensitiveFieldIsNotRedacted() {
RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build();
String result = DebugFormat.multiline().toString(message);
assertThat(result)
.matches(
String.format("%soptional_unredacted_string: \"foo\"\n", UNSTABLE_PREFIX_MULTILINE));
}
@Test
public void descriptorDebugFormat_returnsExpectedFormat() {
FieldDescriptor field =
RedactedFields.getDescriptor().findFieldByName("optional_redacted_string");
String result = DebugFormat.multiline().toString(field, "foo");
assertThat(result)
.matches(
String.format(
"%soptional_redacted_string: %s\n", UNSTABLE_PREFIX_MULTILINE, REDACTED_REGEX));
}
@Test
public void unstableFormat_isStablePerProcess() {
RedactedFields message1 =
RedactedFields.newBuilder().setOptionalUnredactedString("foo").build();
RedactedFields message2 =
RedactedFields.newBuilder().setOptionalUnredactedString("foo").build();
for (int i = 0; i < 5; i++) {
String result1 = DebugFormat.multiline().toString(message1);
String result2 = DebugFormat.multiline().toString(message2);
assertThat(result1).isEqualTo(result2);
}
}
@Test
public void lazyDebugFormatMessage_supportsImplicitFormatting() {
RedactedFields message = RedactedFields.newBuilder().setOptionalUnredactedString("foo").build();
Object lazyDebug = DebugFormat.singleLine().lazyToString(message);
assertThat(String.format("%s", lazyDebug))
.matches(
String.format("%soptional_unredacted_string: \"foo\"", UNSTABLE_PREFIX_SINGLE_LINE));
}
}

6
src/google/protobuf/unittest.proto
View File

@ -1738,6 +1738,12 @@ message RedactedFields {
repeated TestNestedMessageRedaction repeated_unredacted_message = 8;
map<string, string> map_redacted_string = 9 [debug_redact = true];
map<string, string> map_unredacted_string = 10;
optional string optional_redacted_false_string = 11 [debug_redact = false];
extensions 20 to 30;
}
extend RedactedFields {
optional string redacted_extension = 20 [debug_redact = true];
}
message TestCord{