Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: interpret Integer and Float values for TIMESTAMP as microseconds #2175

Merged
merged 2 commits into from
Jul 10, 2023
Merged

fix: interpret Integer and Float values for TIMESTAMP as microseconds #2175

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5e4b3d4
fix: interpret Integer and Float values for TIMESTAMP as microseconds
agrawal-siddharth Jun 28, 2023
7514155
Merge branch 'main' into fixTimestamp
agrawal-siddharth Jul 6, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions ...igquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ private void fillField(
if (val instanceof String) {
Double parsed = Doubles.tryParse((String) val);
if (parsed != null) {
protoMsg.setField(fieldDescriptor, parsed.longValue() * 10000000);
protoMsg.setField(fieldDescriptor, parsed.longValue());
return;
}
TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val);
Expand All @@ -455,7 +455,7 @@ private void fillField(
protoMsg.setField(fieldDescriptor, val);
return;
} else if (val instanceof Integer) {
protoMsg.setField(fieldDescriptor, Long.valueOf((Integer) val) * 10000000);
protoMsg.setField(fieldDescriptor, Long.valueOf((Integer) val));
return;
}
}
Expand Down Expand Up @@ -705,7 +705,7 @@ private void fillRepeatedField(
if (val instanceof String) {
Double parsed = Doubles.tryParse((String) val);
if (parsed != null) {
protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue() * 10000000);
protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue());
} else {
TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val);
protoMsg.addRepeatedField(
Expand All @@ -716,7 +716,7 @@ private void fillRepeatedField(
} else if (val instanceof Long) {
protoMsg.addRepeatedField(fieldDescriptor, val);
} else if (val instanceof Integer) {
protoMsg.addRepeatedField(fieldDescriptor, ((Integer) val) * 10000000);
protoMsg.addRepeatedField(fieldDescriptor, Long.valueOf((Integer) val));
} else {
throwWrongFieldType(fieldDescriptor, currentScope, index);
}
Expand Down
79 changes: 74 additions & 5 deletions ...erystorage/src/test/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessageTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,6 +393,12 @@ public class JsonToProtoMessageTest {
.setMode(TableFieldSchema.Mode.NULLABLE)
.setName("test_timestamp")
.build();
private final TableFieldSchema TEST_TIMESTAMP_REPEATED =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.TIMESTAMP)
.setMode(TableFieldSchema.Mode.REPEATED)
.setName("test_timestamp_repeated")
.build();
private final TableFieldSchema TEST_TIME =
TableFieldSchema.newBuilder()
.setType(TableFieldSchema.Type.TIME)
Expand Down Expand Up @@ -787,17 +793,17 @@ public void testTimestamp() throws Exception {
TestTimestamp.newBuilder()
.setTestString(10L)
.setTestStringTZ(1648493279010000L)
.setTestLong(0L)
.setTestInt(1534806950000000L)
.setTestFloat(1534680695000000000L)
.setTestLong(1687984085000000L)
.setTestInt(153480695L)
.setTestFloat(153468069500L)
.setTestOffset(1649135171000000L)
.setTestTimezone(1649174771000000L)
.setTestSaformat(1534680660000000L)
.build();
JSONObject json = new JSONObject();
json.put("test_string", "1970-01-01 00:00:00.000010");
json.put("test_string_T_Z", "2022-03-28T18:47:59.01Z");
json.put("test_long", 0L);
json.put("test_long", 1687984085000000L);
json.put("test_int", 153480695);
json.put("test_float", "1.534680695e11");
json.put("test_offset", "2022-04-05T09:06:11+04:00");
Expand All @@ -809,6 +815,69 @@ public void testTimestamp() throws Exception {
assertEquals(expectedProto, protoMsg);
}

@Test
public void testTimestampRepeated() throws Exception {
TableSchema tableSchema =
TableSchema.newBuilder()
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_string_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_string_T_Z_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_long_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_int_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_float_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_offset_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_timezone_repeated")
.build())
.addFields(
TableFieldSchema.newBuilder(TEST_TIMESTAMP_REPEATED)
.setName("test_saformat_repeated")
.build())
.build();
TestRepeatedTimestamp expectedProto =
TestRepeatedTimestamp.newBuilder()
.addTestStringRepeated(10L)
.addTestStringTZRepeated(1648493279010000L)
.addTestLongRepeated(1687984085000000L)
.addTestIntRepeated(153480695L)
.addTestFloatRepeated(153468069500L)
.addTestOffsetRepeated(1649135171000000L)
.addTestTimezoneRepeated(1649174771000000L)
.addTestSaformatRepeated(1534680660000000L)
.build();
JSONObject json = new JSONObject();
json.put("test_string_repeated", new JSONArray(new String[] {"1970-01-01 00:00:00.000010"}));
json.put("test_string_T_Z_repeated", new JSONArray(new String[] {"2022-03-28T18:47:59.01Z"}));
json.put("test_long_repeated", new JSONArray(new Long[] {1687984085000000L}));
json.put("test_int_repeated", new JSONArray(new Integer[] {153480695}));
json.put("test_float_repeated", new JSONArray(new String[] {"1.534680695e11"}));
json.put("test_offset_repeated", new JSONArray(new String[] {"2022-04-05T09:06:11+04:00"}));
json.put("test_timezone_repeated", new JSONArray(new String[] {"2022-04-05 09:06:11 PST"}));
json.put("test_saformat_repeated", new JSONArray(new String[] {"2018/08/19 12:11"}));
DynamicMessage protoMsg =
JsonToProtoMessage.INSTANCE.convertToProtoMessage(
TestRepeatedTimestamp.getDescriptor(), tableSchema, json);
assertEquals(expectedProto, protoMsg);
}

@Test
public void testDate() throws Exception {
TableSchema tableSchema =
Expand Down Expand Up @@ -993,7 +1062,7 @@ public void testStructComplex() throws Exception {
.setTestNumeric(
BigDecimalByteStringEncoder.encodeToNumericByteString(new BigDecimal("1.23456")))
.setTestGeo("POINT(1,1)")
.setTestTimestamp(123456780000000L)
.setTestTimestamp(12345678L)
.setTestTime(CivilTimeEncoder.encodePacked64TimeMicros(LocalTime.of(1, 0, 1)))
.setTestTimeStr(89332507144L)
.addTestNumericRepeated(
Expand Down
11 changes: 11 additions & 0 deletions google-cloud-bigquerystorage/src/test/proto/jsonTest.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,17 @@ message TestTimestamp {
optional int64 test_saformat = 8;
}

message TestRepeatedTimestamp {
repeated int64 test_string_repeated = 1;
repeated int64 test_string_t_z_repeated = 2;
repeated int64 test_long_repeated = 3;
repeated int64 test_int_repeated = 4;
repeated int64 test_float_repeated = 5;
repeated int64 test_offset_repeated = 6;
repeated int64 test_timezone_repeated = 7;
repeated int64 test_saformat_repeated = 8;
}

message TestDate {
optional int32 test_string = 1;
optional int32 test_long = 2;
Expand Down