feat: Add support for Postgresql dialect (#741)

* chore: regen (via synth) : fix conflicts

* feat: add NUMERIC support: conflicts resolved

* feat: add dialect support: fix conflicts

* fix: update table queries to support PG dialect

* feat: add database dialect support for database factory

* test: add dialect support to system tests: resolve conflict, correct POSTGRES_ALL_TYPES_COLUMNS

* feat: postgres dialect - review fixes

* feat: postgres dialect - review fixes

* feat: postgres dialect - review fixes

* feat: postgres dialect - review fixes

* feat: postgres dialect - review fixes

* feat: postgres dialect - review fixes

* feat: postgres dialect - review fixes

* feat: postgres dialect - review fixes

* feat: postgres dialect - docstring fixes

* feat: fix linting

* feat: add opentelemetry version in noxfile to remove failures

* feat: add opentelemetry version and constraints.txt

* Revert "feat: add opentelemetry version and constraints.txt"

This reverts commit 8525bf5.

* Revert "feat: add opentelemetry version in noxfile to remove failures"

This reverts commit 666285b.

* feat: removing duplicate imports

* feat: correcting imports

* feat: correcting imports

* feat: skip backup tests

* feat: correct the import

* feat: fix linting

Co-authored-by: larkee <[email protected]>

Author: ansh0l

google/cloud/spanner_admin_database_v1/types/spanner_database_admin.py

@@ -294,7 +294,7 @@ class CreateDatabaseRequest(proto.Message):
             Cloud Spanner will encrypt/decrypt all data at
             rest using Google default encryption.
         database_dialect (google.cloud.spanner_admin_database_v1.types.DatabaseDialect):
-            Optional. The dialect of the Cloud Spanner
+            Output only. The dialect of the Cloud Spanner
             Database.
     """
 

google/cloud/spanner_v1/__init__.py

@@ -57,6 +57,7 @@
 from .types.transaction import TransactionSelector
 from .types.type import StructType
 from .types.type import Type
+from .types.type import TypeAnnotationCode
 from .types.type import TypeCode
 from .data_types import JsonObject
 
@@ -132,6 +133,7 @@
     "TransactionOptions",
     "TransactionSelector",
     "Type",
+    "TypeAnnotationCode",
     "TypeCode",
     # Custom spanner related data types
     "JsonObject",

google/cloud/spanner_v1/backup.py

@@ -94,6 +94,7 @@ def __init__(
         self._encryption_info = None
         self._max_expire_time = None
         self._referencing_backups = None
+        self._database_dialect = None
         if type(encryption_config) == dict:
             if source_backup:
                 self._encryption_config = CopyBackupEncryptionConfig(
@@ -193,7 +194,7 @@ def referencing_databases(self):
     @property
     def encryption_info(self):
         """Encryption info for this backup.
-        :rtype: :class:`~google.clod.spanner_admin_database_v1.types.EncryptionInfo`
+        :rtype: :class:`~google.cloud.spanner_admin_database_v1.types.EncryptionInfo`
         :returns: a class representing the encryption info
         """
         return self._encryption_info
@@ -216,6 +217,13 @@ def referencing_backups(self):
         """
         return self._referencing_backups
 
+    def database_dialect(self):
+        """Database Dialect for this backup.
+        :rtype: :class:`~google.cloud.spanner_admin_database_v1.types.DatabaseDialect`
+        :returns: a class representing the dialect of this backup's database
+        """
+        return self._database_dialect
+
     @classmethod
     def from_pb(cls, backup_pb, instance):
         """Create an instance of this class from a protobuf message.

google/cloud/spanner_v1/database.py

@@ -34,6 +34,7 @@
 from google.cloud.spanner_admin_database_v1 import RestoreDatabaseEncryptionConfig
 from google.cloud.spanner_admin_database_v1 import RestoreDatabaseRequest
 from google.cloud.spanner_admin_database_v1 import UpdateDatabaseDdlRequest
+from google.cloud.spanner_admin_database_v1.types import DatabaseDialect
 from google.cloud.spanner_v1 import ExecuteSqlRequest
 from google.cloud.spanner_v1 import TransactionSelector
 from google.cloud.spanner_v1 import TransactionOptions
@@ -68,7 +69,7 @@
 
 _LIST_TABLES_QUERY = """SELECT TABLE_NAME
 FROM INFORMATION_SCHEMA.TABLES
-WHERE SPANNER_STATE = 'COMMITTED'
+{}
 """
 
 DEFAULT_RETRY_BACKOFF = Retry(initial=0.02, maximum=32, multiplier=1.3)
@@ -114,6 +115,11 @@ class Database(object):
         If a dict is provided, it must be of the same form as either of the protobuf
         messages :class:`~google.cloud.spanner_admin_database_v1.types.EncryptionConfig`
         or :class:`~google.cloud.spanner_admin_database_v1.types.RestoreDatabaseEncryptionConfig`
+    :type database_dialect:
+        :class:`~google.cloud.spanner_admin_database_v1.types.DatabaseDialect`
+    :param database_dialect:
+        (Optional) database dialect for the database
+
     """
 
     _spanner_api = None
@@ -126,6 +132,7 @@ def __init__(
         pool=None,
         logger=None,
         encryption_config=None,
+        database_dialect=DatabaseDialect.DATABASE_DIALECT_UNSPECIFIED,
     ):
         self.database_id = database_id
         self._instance = instance
@@ -141,6 +148,7 @@ def __init__(
         self.log_commit_stats = False
         self._logger = logger
         self._encryption_config = encryption_config
+        self._database_dialect = database_dialect
 
         if pool is None:
             pool = BurstyPool()
@@ -294,6 +302,18 @@ def ddl_statements(self):
         """
         return self._ddl_statements
 
+    @property
+    def database_dialect(self):
+        """DDL Statements used to define database schema.
+
+        See
+        cloud.google.com/spanner/docs/data-definition-language
+
+        :rtype: :class:`google.cloud.spanner_admin_database_v1.types.DatabaseDialect`
+        :returns: the dialect of the database
+        """
+        return self._database_dialect
+
     @property
     def logger(self):
         """Logger used by the database.
@@ -364,7 +384,10 @@ def create(self):
         metadata = _metadata_with_prefix(self.name)
         db_name = self.database_id
         if "-" in db_name:
-            db_name = "`%s`" % (db_name,)
+            if self._database_dialect == DatabaseDialect.POSTGRESQL:
+                db_name = f'"{db_name}"'
+            else:
+                db_name = f"`{db_name}`"
         if type(self._encryption_config) == dict:
             self._encryption_config = EncryptionConfig(**self._encryption_config)
 
@@ -373,6 +396,7 @@ def create(self):
             create_statement="CREATE DATABASE %s" % (db_name,),
             extra_statements=list(self._ddl_statements),
             encryption_config=self._encryption_config,
+            database_dialect=self._database_dialect,
         )
         future = api.create_database(request=request, metadata=metadata)
         return future
@@ -418,6 +442,7 @@ def reload(self):
         self._encryption_config = response.encryption_config
         self._encryption_info = response.encryption_info
         self._default_leader = response.default_leader
+        self._database_dialect = response.database_dialect
 
     def update_ddl(self, ddl_statements, operation_id=""):
         """Update DDL for this database.
@@ -778,7 +803,11 @@ def list_tables(self):
             resources within the current database.
         """
         with self.snapshot() as snapshot:
-            results = snapshot.execute_sql(_LIST_TABLES_QUERY)
+            if self._database_dialect == DatabaseDialect.POSTGRESQL:
+                where_clause = "WHERE TABLE_SCHEMA = 'public'"
+            else:
+                where_clause = "WHERE SPANNER_STATE = 'COMMITTED'"
+            results = snapshot.execute_sql(_LIST_TABLES_QUERY.format(where_clause))
             for row in results:
                 yield self.table(row[0])
 

google/cloud/spanner_v1/instance.py

@@ -25,6 +25,7 @@
 from google.cloud.spanner_admin_instance_v1 import Instance as InstancePB
 from google.cloud.spanner_admin_database_v1.types import backup
 from google.cloud.spanner_admin_database_v1.types import spanner_database_admin
+from google.cloud.spanner_admin_database_v1 import DatabaseDialect
 from google.cloud.spanner_admin_database_v1 import ListBackupsRequest
 from google.cloud.spanner_admin_database_v1 import ListBackupOperationsRequest
 from google.cloud.spanner_admin_database_v1 import ListDatabasesRequest
@@ -428,6 +429,7 @@ def database(
         pool=None,
         logger=None,
         encryption_config=None,
+        database_dialect=DatabaseDialect.DATABASE_DIALECT_UNSPECIFIED,
     ):
         """Factory to create a database within this instance.
 
@@ -458,6 +460,11 @@ def database(
             messages :class:`~google.cloud.spanner_admin_database_v1.types.EncryptionConfig`
             or :class:`~google.cloud.spanner_admin_database_v1.types.RestoreDatabaseEncryptionConfig`
 
+        :type database_dialect:
+            :class:`~google.cloud.spanner_admin_database_v1.types.DatabaseDialect`
+        :param database_dialect:
+            (Optional) database dialect for the database
+
         :rtype: :class:`~google.cloud.spanner_v1.database.Database`
         :returns: a database owned by this instance.
         """
@@ -468,6 +475,7 @@ def database(
             pool=pool,
             logger=logger,
             encryption_config=encryption_config,
+            database_dialect=database_dialect,
         )
 
     def list_databases(self, page_size=None):

google/cloud/spanner_v1/param_types.py

@@ -15,6 +15,7 @@
 """Types exported from this package."""
 
 from google.cloud.spanner_v1 import Type
+from google.cloud.spanner_v1 import TypeAnnotationCode
 from google.cloud.spanner_v1 import TypeCode
 from google.cloud.spanner_v1 import StructType
 
@@ -29,6 +30,7 @@
 TIMESTAMP = Type(code=TypeCode.TIMESTAMP)
 NUMERIC = Type(code=TypeCode.NUMERIC)
 JSON = Type(code=TypeCode.JSON)
+PG_NUMERIC = Type(code=TypeCode.NUMERIC, type_annotation=TypeAnnotationCode.PG_NUMERIC)
 
 
 def Array(element_type):

google/cloud/spanner_v1/table.py

@@ -16,6 +16,7 @@
 
 from google.cloud.exceptions import NotFound
 
+from google.cloud.spanner_admin_database_v1 import DatabaseDialect
 from google.cloud.spanner_v1.types import (
     Type,
     TypeCode,
@@ -26,7 +27,7 @@
 SELECT EXISTS(
     SELECT TABLE_NAME
     FROM INFORMATION_SCHEMA.TABLES
-    WHERE TABLE_NAME = @table_id
+    {}
 )
 """
 _GET_SCHEMA_TEMPLATE = "SELECT * FROM {} LIMIT 0"
@@ -76,11 +77,18 @@ def _exists(self, snapshot):
         :rtype: bool
         :returns: True if the table exists, else false.
         """
-        results = snapshot.execute_sql(
-            _EXISTS_TEMPLATE,
-            params={"table_id": self.table_id},
-            param_types={"table_id": Type(code=TypeCode.STRING)},
-        )
+        if self._database.database_dialect == DatabaseDialect.POSTGRESQL:
+            results = snapshot.execute_sql(
+                _EXISTS_TEMPLATE.format("WHERE TABLE_NAME = $1"),
+                params={"p1": self.table_id},
+                param_types={"p1": Type(code=TypeCode.STRING)},
+            )
+        else:
+            results = snapshot.execute_sql(
+                _EXISTS_TEMPLATE.format("WHERE TABLE_NAME = @table_id"),
+                params={"table_id": self.table_id},
+                param_types={"table_id": Type(code=TypeCode.STRING)},
+            )
         return next(iter(results))[0]
 
     @property