fix: invalid conversion of timezone-aware datetime values to JSON (#480)

plamut · web-flow · commit 61b438523d30 · 2021-01-25T18:04:04.000+01:00
* fix: correctly convert timezone-aware datetimes

* blacken

* Remove python-dateutil test dependency

* Remove unused dst() methods
diff --git a/google/cloud/bigquery/_helpers.py b/google/cloud/bigquery/_helpers.py
@@ -315,13 +315,21 @@ def _timestamp_to_json_parameter(value):
 def _timestamp_to_json_row(value):
     """Coerce 'value' to an JSON-compatible representation."""
     if isinstance(value, datetime.datetime):
+        # For naive datetime objects UTC timezone is assumed, thus we format
+        # those to string directly without conversion.
+        if value.tzinfo is not None:
+            value = value.astimezone(UTC)
         value = value.strftime(_RFC3339_MICROS)
     return value
 
 
 def _datetime_to_json(value):
     """Coerce 'value' to an JSON-compatible representation."""
     if isinstance(value, datetime.datetime):
+        # For naive datetime objects UTC timezone is assumed, thus we format
+        # those to string directly without conversion.
+        if value.tzinfo is not None:
+            value = value.astimezone(UTC)
         value = value.strftime(_RFC3339_MICROS_NO_ZULU)
     return value
 
diff --git a/tests/unit/test__helpers.py b/tests/unit/test__helpers.py
@@ -420,27 +420,27 @@ def _call_fut(self, row, schema):
     def test_w_single_scalar_column(self):
         # SELECT 1 AS col
         col = _Field("REQUIRED", "col", "INTEGER")
-        row = {u"f": [{u"v": u"1"}]}
+        row = {"f": [{"v": "1"}]}
         self.assertEqual(self._call_fut(row, schema=[col]), (1,))
 
     def test_w_single_scalar_geography_column(self):
         # SELECT 1 AS col
         col = _Field("REQUIRED", "geo", "GEOGRAPHY")
-        row = {u"f": [{u"v": u"POINT(1, 2)"}]}
+        row = {"f": [{"v": "POINT(1, 2)"}]}
         self.assertEqual(self._call_fut(row, schema=[col]), ("POINT(1, 2)",))
 
     def test_w_single_struct_column(self):
         # SELECT (1, 2) AS col
         sub_1 = _Field("REQUIRED", "sub_1", "INTEGER")
         sub_2 = _Field("REQUIRED", "sub_2", "INTEGER")
         col = _Field("REQUIRED", "col", "RECORD", fields=[sub_1, sub_2])
-        row = {u"f": [{u"v": {u"f": [{u"v": u"1"}, {u"v": u"2"}]}}]}
+        row = {"f": [{"v": {"f": [{"v": "1"}, {"v": "2"}]}}]}
         self.assertEqual(self._call_fut(row, schema=[col]), ({"sub_1": 1, "sub_2": 2},))
 
     def test_w_single_array_column(self):
         # SELECT [1, 2, 3] as col
         col = _Field("REPEATED", "col", "INTEGER")
-        row = {u"f": [{u"v": [{u"v": u"1"}, {u"v": u"2"}, {u"v": u"3"}]}]}
+        row = {"f": [{"v": [{"v": "1"}, {"v": "2"}, {"v": "3"}]}]}
         self.assertEqual(self._call_fut(row, schema=[col]), ([1, 2, 3],))
 
     def test_w_struct_w_nested_array_column(self):
@@ -450,21 +450,21 @@ def test_w_struct_w_nested_array_column(self):
         third = _Field("REPEATED", "third", "INTEGER")
         col = _Field("REQUIRED", "col", "RECORD", fields=[first, second, third])
         row = {
-            u"f": [
+            "f": [
                 {
-                    u"v": {
-                        u"f": [
-                            {u"v": [{u"v": u"1"}, {u"v": u"2"}]},
-                            {u"v": u"3"},
-                            {u"v": [{u"v": u"4"}, {u"v": u"5"}]},
+                    "v": {
+                        "f": [
+                            {"v": [{"v": "1"}, {"v": "2"}]},
+                            {"v": "3"},
+                            {"v": [{"v": "4"}, {"v": "5"}]},
                         ]
                     }
                 }
             ]
         }
         self.assertEqual(
             self._call_fut(row, schema=[col]),
-            ({u"first": [1, 2], u"second": 3, u"third": [4, 5]},),
+            ({"first": [1, 2], "second": 3, "third": [4, 5]},),
         )
 
     def test_w_array_of_struct(self):
@@ -474,11 +474,11 @@ def test_w_array_of_struct(self):
         third = _Field("REQUIRED", "third", "INTEGER")
         col = _Field("REPEATED", "col", "RECORD", fields=[first, second, third])
         row = {
-            u"f": [
+            "f": [
                 {
-                    u"v": [
-                        {u"v": {u"f": [{u"v": u"1"}, {u"v": u"2"}, {u"v": u"3"}]}},
-                        {u"v": {u"f": [{u"v": u"4"}, {u"v": u"5"}, {u"v": u"6"}]}},
+                    "v": [
+                        {"v": {"f": [{"v": "1"}, {"v": "2"}, {"v": "3"}]}},
+                        {"v": {"f": [{"v": "4"}, {"v": "5"}, {"v": "6"}]}},
                     ]
                 }
             ]
@@ -487,8 +487,8 @@ def test_w_array_of_struct(self):
             self._call_fut(row, schema=[col]),
             (
                 [
-                    {u"first": 1, u"second": 2, u"third": 3},
-                    {u"first": 4, u"second": 5, u"third": 6},
+                    {"first": 1, "second": 2, "third": 3},
+                    {"first": 4, "second": 5, "third": 6},
                 ],
             ),
         )
@@ -499,32 +499,25 @@ def test_w_array_of_struct_w_array(self):
         second = _Field("REQUIRED", "second", "INTEGER")
         col = _Field("REPEATED", "col", "RECORD", fields=[first, second])
         row = {
-            u"f": [
+            "f": [
                 {
-                    u"v": [
-                        {
-                            u"v": {
-                                u"f": [
-                                    {u"v": [{u"v": u"1"}, {u"v": u"2"}, {u"v": u"3"}]},
-                                    {u"v": u"4"},
-                                ]
-                            }
-                        },
+                    "v": [
                         {
-                            u"v": {
-                                u"f": [
-                                    {u"v": [{u"v": u"5"}, {u"v": u"6"}]},
-                                    {u"v": u"7"},
+                            "v": {
+                                "f": [
+                                    {"v": [{"v": "1"}, {"v": "2"}, {"v": "3"}]},
+                                    {"v": "4"},
                                 ]
                             }
                         },
+                        {"v": {"f": [{"v": [{"v": "5"}, {"v": "6"}]}, {"v": "7"}]}},
                     ]
                 }
             ]
         }
         self.assertEqual(
             self._call_fut(row, schema=[col]),
-            ([{u"first": [1, 2, 3], u"second": 4}, {u"first": [5, 6], u"second": 7}],),
+            ([{"first": [1, 2, 3], "second": 4}, {"first": [5, 6], "second": 7}],),
         )
 
 
@@ -673,7 +666,7 @@ def test_w_non_bytes(self):
 
     def test_w_bytes(self):
         source = b"source"
-        expected = u"c291cmNl"
+        expected = "c291cmNl"
         converted = self._call_fut(source)
         self.assertEqual(converted, expected)
 
@@ -726,7 +719,7 @@ def test_w_string(self):
         ZULU = "2016-12-20 15:58:27.339328+00:00"
         self.assertEqual(self._call_fut(ZULU), ZULU)
 
-    def test_w_datetime(self):
+    def test_w_datetime_no_zone(self):
         when = datetime.datetime(2016, 12, 20, 15, 58, 27, 339328)
         self.assertEqual(self._call_fut(when), "2016-12-20T15:58:27.339328Z")
 
@@ -736,6 +729,14 @@ def test_w_datetime_w_utc_zone(self):
         when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=UTC)
         self.assertEqual(self._call_fut(when), "2020-11-17T01:06:52.353795Z")
 
+    def test_w_datetime_w_non_utc_zone(self):
+        class EstZone(datetime.tzinfo):
+            def utcoffset(self, _):
+                return datetime.timedelta(minutes=-300)
+
+        when = datetime.datetime(2020, 11, 17, 1, 6, 52, 353795, tzinfo=EstZone())
+        self.assertEqual(self._call_fut(when), "2020-11-17T06:06:52.353795Z")
+
 
 class Test_datetime_to_json(unittest.TestCase):
     def _call_fut(self, value):
@@ -753,6 +754,14 @@ def test_w_datetime(self):
         when = datetime.datetime(2016, 12, 3, 14, 11, 27, 123456, tzinfo=UTC)
         self.assertEqual(self._call_fut(when), "2016-12-03T14:11:27.123456")
 
+    def test_w_datetime_w_non_utc_zone(self):
+        class EstZone(datetime.tzinfo):
+            def utcoffset(self, _):
+                return datetime.timedelta(minutes=-300)
+
+        when = datetime.datetime(2016, 12, 3, 14, 11, 27, 123456, tzinfo=EstZone())
+        self.assertEqual(self._call_fut(when), "2016-12-03T19:11:27.123456")
+
 
 class Test_date_to_json(unittest.TestCase):
     def _call_fut(self, value):
