diff --git a/.flake8 b/.flake8 index ed931638..29227d4c 100644 --- a/.flake8 +++ b/.flake8 @@ -26,6 +26,7 @@ exclude = *_pb2.py # Standard linting exemptions. + **/.nox/** __pycache__, .git, *.pyc, diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..a9024b15 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml +- repo: https://github.com/psf/black + rev: 19.10b0 + hooks: + - id: black +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.4 + hooks: + - id: flake8 diff --git a/CHANGELOG.md b/CHANGELOG.md index 42c2c620..80f76e2e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://pypi.org/project/google-cloud-dlp/#history +### [3.0.1](https://www.github.com/googleapis/python-dlp/compare/v3.0.0...v3.0.1) (2021-01-28) + + +### Bug Fixes + +* remove gRPC send/recv limits; add enums to `types/__init__.py` ([#89](https://www.github.com/googleapis/python-dlp/issues/89)) ([76e0439](https://www.github.com/googleapis/python-dlp/commit/76e0439b3acfdacf9303595107c03c1d49eac8b6)) + ## [3.0.0](https://www.github.com/googleapis/python-dlp/compare/v2.0.0...v3.0.0) (2020-12-02) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 87ac6da7..c2c37699 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -21,8 +21,8 @@ In order to add a feature: - The feature must be documented in both the API and narrative documentation. -- The feature must work fully on the following CPython versions: 2.7, - 3.5, 3.6, 3.7 and 3.8 on both UNIX and Windows. +- The feature must work fully on the following CPython versions: + 3.6, 3.7, 3.8 and 3.9 on both UNIX and Windows. - The feature must not add unnecessary dependencies (where "unnecessary" is of course subjective, but new dependencies should @@ -111,6 +111,16 @@ Coding Style should point to the official ``googleapis`` checkout and the the branch should be the main branch on that remote (``master``). +- This repository contains configuration for the + `pre-commit `__ tool, which automates checking + our linters during a commit. If you have it installed on your ``$PATH``, + you can enable enforcing those checks via: + +.. code-block:: bash + + $ pre-commit install + pre-commit installed at .git/hooks/pre-commit + Exceptions to PEP8: - Many unit tests use a helper method, ``_call_fut`` ("FUT" is short for @@ -192,25 +202,24 @@ Supported Python Versions We support: -- `Python 3.5`_ - `Python 3.6`_ - `Python 3.7`_ - `Python 3.8`_ +- `Python 3.9`_ -.. _Python 3.5: https://docs.python.org/3.5/ .. _Python 3.6: https://docs.python.org/3.6/ .. _Python 3.7: https://docs.python.org/3.7/ .. _Python 3.8: https://docs.python.org/3.8/ +.. _Python 3.9: https://docs.python.org/3.9/ Supported versions can be found in our ``noxfile.py`` `config`_. .. _config: https://github.com/googleapis/python-dlp/blob/master/noxfile.py -Python 2.7 support is deprecated. All code changes should maintain Python 2.7 compatibility until January 1, 2020. We also explicitly decided to support Python 3 beginning with version -3.5. Reasons for this include: +3.6. Reasons for this include: - Encouraging use of newest versions of Python 3 - Taking the lead of `prominent`_ open-source `projects`_ diff --git a/docs/conf.py b/docs/conf.py index b6f3b255..10dd0aa8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -339,10 +339,10 @@ # Example configuration for intersphinx: refer to the Python standard library. intersphinx_mapping = { - "python": ("http://python.readthedocs.org/en/latest/", None), - "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "python": ("https://python.readthedocs.org/en/latest/", None), + "google-auth": ("https://googleapis.dev/python/google-auth/latest/", None), "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None,), - "grpc": ("https://grpc.io/grpc/python/", None), + "grpc": ("https://grpc.github.io/grpc/python/", None), "proto-plus": ("https://proto-plus-python.readthedocs.io/en/latest/", None), } diff --git a/google/cloud/dlp_v2/services/dlp_service/transports/__init__.py b/google/cloud/dlp_v2/services/dlp_service/transports/__init__.py index 983401ce..fcd0f4ba 100644 --- a/google/cloud/dlp_v2/services/dlp_service/transports/__init__.py +++ b/google/cloud/dlp_v2/services/dlp_service/transports/__init__.py @@ -28,7 +28,6 @@ _transport_registry["grpc"] = DlpServiceGrpcTransport _transport_registry["grpc_asyncio"] = DlpServiceGrpcAsyncIOTransport - __all__ = ( "DlpServiceTransport", "DlpServiceGrpcTransport", diff --git a/google/cloud/dlp_v2/services/dlp_service/transports/grpc.py b/google/cloud/dlp_v2/services/dlp_service/transports/grpc.py index 0a0cb81b..f7ed34a4 100644 --- a/google/cloud/dlp_v2/services/dlp_service/transports/grpc.py +++ b/google/cloud/dlp_v2/services/dlp_service/transports/grpc.py @@ -155,6 +155,10 @@ def __init__( ssl_credentials=ssl_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._ssl_channel_credentials = ssl_credentials else: @@ -173,6 +177,10 @@ def __init__( ssl_credentials=ssl_channel_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._stubs = {} # type: Dict[str, Callable] @@ -199,7 +207,7 @@ def create_channel( ) -> grpc.Channel: """Create and return a gRPC channel object. Args: - address (Optionsl[str]): The host for the channel to use. + address (Optional[str]): The host for the channel to use. credentials (Optional[~.Credentials]): The authorization credentials to attach to requests. These credentials identify this application to the service. If diff --git a/google/cloud/dlp_v2/services/dlp_service/transports/grpc_asyncio.py b/google/cloud/dlp_v2/services/dlp_service/transports/grpc_asyncio.py index 27b0f50c..41654b8a 100644 --- a/google/cloud/dlp_v2/services/dlp_service/transports/grpc_asyncio.py +++ b/google/cloud/dlp_v2/services/dlp_service/transports/grpc_asyncio.py @@ -200,6 +200,10 @@ def __init__( ssl_credentials=ssl_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) self._ssl_channel_credentials = ssl_credentials else: @@ -218,6 +222,10 @@ def __init__( ssl_credentials=ssl_channel_credentials, scopes=scopes or self.AUTH_SCOPES, quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) # Run the base constructor. diff --git a/google/cloud/dlp_v2/types/__init__.py b/google/cloud/dlp_v2/types/__init__.py index 32b59e7f..5e17edb4 100644 --- a/google/cloud/dlp_v2/types/__init__.py +++ b/google/cloud/dlp_v2/types/__init__.py @@ -38,6 +38,8 @@ BigQueryField, EntityId, TableOptions, + Likelihood, + FileType, ) from .dlp import ( ExcludeInfoTypes, @@ -164,9 +166,15 @@ HybridContentItem, HybridFindingDetails, HybridInspectResponse, + RelationalOperator, + MatchingType, + ContentOption, + MetadataType, + InfoTypeSupportedBy, + DlpJobType, + StoredInfoTypeState, ) - __all__ = ( "InfoType", "StoredType", @@ -190,6 +198,8 @@ "BigQueryField", "EntityId", "TableOptions", + "Likelihood", + "FileType", "ExcludeInfoTypes", "ExclusionRule", "InspectionRule", @@ -314,4 +324,11 @@ "HybridContentItem", "HybridFindingDetails", "HybridInspectResponse", + "RelationalOperator", + "MatchingType", + "ContentOption", + "MetadataType", + "InfoTypeSupportedBy", + "DlpJobType", + "StoredInfoTypeState", ) diff --git a/noxfile.py b/noxfile.py index b776ccc5..25452f0b 100644 --- a/noxfile.py +++ b/noxfile.py @@ -81,9 +81,8 @@ def default(session): session.run( "py.test", "--quiet", - "--cov=google.cloud.dlp", - "--cov=google.cloud", - "--cov=tests.unit", + "--cov=google/cloud", + "--cov=tests/unit", "--cov-append", "--cov-config=.coveragerc", "--cov-report=", diff --git a/samples/snippets/README.rst b/samples/snippets/README.rst deleted file mode 100644 index 0b25cc7a..00000000 --- a/samples/snippets/README.rst +++ /dev/null @@ -1,405 +0,0 @@ - -.. This file is automatically generated. Do not edit this file directly. - -Google Data Loss Prevention Python Samples -=============================================================================== - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/README.rst - - -This directory contains samples for Google Data Loss Prevention. `Google Data Loss Prevention`_ provides programmatic access to a powerful detection engine for personally identifiable information and other privacy-sensitive data in unstructured data streams. - - - - -.. _Google Data Loss Prevention: https://cloud.google.com/dlp/docs/ - - -Setup -------------------------------------------------------------------------------- - - - -Authentication -++++++++++++++ - -This sample requires you to have authentication setup. Refer to the -`Authentication Getting Started Guide`_ for instructions on setting up -credentials for applications. - -.. _Authentication Getting Started Guide: - https://cloud.google.com/docs/authentication/getting-started - - - - -Install Dependencies -++++++++++++++++++++ - -#. Clone python-docs-samples and change directory to the sample directory you want to use. - - .. code-block:: bash - - $ git clone https://github.com/GoogleCloudPlatform/python-docs-samples.git - -#. Install `pip`_ and `virtualenv`_ if you do not already have them. You may want to refer to the `Python Development Environment Setup Guide`_ for Google Cloud Platform for instructions. - - .. _Python Development Environment Setup Guide: - https://cloud.google.com/python/setup - -#. Create a virtualenv. Samples are compatible with Python 3.6+. - - .. code-block:: bash - - $ virtualenv env - $ source env/bin/activate - -#. Install the dependencies needed to run the samples. - - .. code-block:: bash - - $ pip install -r requirements.txt - -.. _pip: https://pip.pypa.io/ -.. _virtualenv: https://virtualenv.pypa.io/ - - - - - - -Samples -------------------------------------------------------------------------------- - - -Quickstart -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/quickstart.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python quickstart.py - - - - -Inspect Content -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/inspect_content.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python inspect_content.py - - - usage: inspect_content.py [-h] {string,table,file,gcs,datastore,bigquery} ... - - Sample app that uses the Data Loss Prevention API to inspect a string, a local - file or a file on Google Cloud Storage. - - positional arguments: - {string,table,file,gcs,datastore,bigquery} - Select how to submit content to the API. - string Inspect a string. - table Inspect a table. - file Inspect a local file. - gcs Inspect files on Google Cloud Storage. - datastore Inspect files on Google Datastore. - bigquery Inspect files on Google BigQuery. - - optional arguments: - -h, --help show this help message and exit - - - - - -Redact Content -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/redact.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python redact.py - - - usage: redact.py [-h] {info_types,all_text} ... - - Sample app that uses the Data Loss Prevent API to redact the contents of an - image file. - - positional arguments: - {info_types,all_text} - Select which content should be redacted. - info_types Redact specific infoTypes from an image. - all_text Redact all text from an image. The MIME type of the - file is inferred via the Python standard library's - mimetypes module. - - optional arguments: - -h, --help show this help message and exit - - - - - -Metadata -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/metadata.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python metadata.py - - - usage: metadata.py [-h] [--language_code LANGUAGE_CODE] [--filter FILTER] - - Sample app that queries the Data Loss Prevention API for supported categories - and info types. - - optional arguments: - -h, --help show this help message and exit - --language_code LANGUAGE_CODE - The BCP-47 language code to use, e.g. 'en-US'. - --filter FILTER An optional filter to only return info types supported - by certain parts of the API. Defaults to - "supported_by=INSPECT". - - - - - -Jobs -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/jobs.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python jobs.py - - - usage: jobs.py [-h] {list,delete} ... - - Sample app to list and delete DLP jobs using the Data Loss Prevent API. - - positional arguments: - {list,delete} Select how to submit content to the API. - list List Data Loss Prevention API jobs corresponding to a given - filter. - delete Delete results of a Data Loss Prevention API job. - - optional arguments: - -h, --help show this help message and exit - - - - - -Templates -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/templates.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python templates.py - - - usage: templates.py [-h] {create,list,delete} ... - - Sample app that sets up Data Loss Prevention API inspect templates. - - positional arguments: - {create,list,delete} Select which action to perform. - create Create a template. - list List all templates. - delete Delete a template. - - optional arguments: - -h, --help show this help message and exit - - - - - -Triggers -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/triggers.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python triggers.py - - - usage: triggers.py [-h] {create,list,delete} ... - - Sample app that sets up Data Loss Prevention API automation triggers. - - positional arguments: - {create,list,delete} Select which action to perform. - create Create a trigger. - list List all triggers. - delete Delete a trigger. - - optional arguments: - -h, --help show this help message and exit - - - - - -Risk Analysis -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/risk.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python risk.py - - - usage: risk.py [-h] {numerical,categorical,k_anonymity,l_diversity,k_map} ... - - Sample app that uses the Data Loss Prevent API to perform risk anaylsis. - - positional arguments: - {numerical,categorical,k_anonymity,l_diversity,k_map} - Select how to submit content to the API. - numerical - categorical - k_anonymity Computes the k-anonymity of a column set in a Google - BigQuerytable. - l_diversity Computes the l-diversity of a column set in a Google - BigQuerytable. - k_map Computes the k-map risk estimation of a column set in - a GoogleBigQuery table. - - optional arguments: - -h, --help show this help message and exit - - - - - -DeID -+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ - -.. image:: https://gstatic.com/cloudssh/images/open-btn.png - :target: https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/GoogleCloudPlatform/python-docs-samples&page=editor&open_in_editor=dlp/deid.py,dlp/README.rst - - - - -To run this sample: - -.. code-block:: bash - - $ python deid.py - - - usage: deid.py [-h] - {deid_mask,deid_replace,deid_fpe,reid_fpe,deid_date_shift,replace_with_infotype} - ... - - Uses of the Data Loss Prevention API for deidentifying sensitive data. - - positional arguments: - {deid_mask,deid_replace,deid_fpe,reid_fpe,deid_date_shift,replace_with_infotype} - Select how to submit content to the API. - deid_mask Deidentify sensitive data in a string by masking it - with a character. - deid_replace Deidentify sensitive data in a string by replacing it - with another string. - deid_fpe Deidentify sensitive data in a string using Format - Preserving Encryption (FPE). - reid_fpe Reidentify sensitive data in a string using Format - Preserving Encryption (FPE). - deid_date_shift Deidentify dates in a CSV file by pseudorandomly - shifting them. - replace_with_infotype - Deidentify sensitive data in a string by replacing it - with the info type of the data. - - optional arguments: - -h, --help show this help message and exit - - - - - - - - - -The client library -------------------------------------------------------------------------------- - -This sample uses the `Google Cloud Client Library for Python`_. -You can read the documentation for more details on API usage and use GitHub -to `browse the source`_ and `report issues`_. - -.. _Google Cloud Client Library for Python: - https://googlecloudplatform.github.io/google-cloud-python/ -.. _browse the source: - https://github.com/GoogleCloudPlatform/google-cloud-python -.. _report issues: - https://github.com/GoogleCloudPlatform/google-cloud-python/issues - - - -.. _Google Cloud SDK: https://cloud.google.com/sdk/ diff --git a/samples/snippets/inspect_content_test.py b/samples/snippets/inspect_content_test.py index c7ba8909..e77d3aee 100644 --- a/samples/snippets/inspect_content_test.py +++ b/samples/snippets/inspect_content_test.py @@ -84,13 +84,13 @@ def topic_id(): publisher = google.cloud.pubsub.PublisherClient() topic_path = publisher.topic_path(GCLOUD_PROJECT, TOPIC_ID) try: - publisher.create_topic(topic_path) + publisher.create_topic(request={"name": topic_path}) except google.api_core.exceptions.AlreadyExists: pass yield TOPIC_ID - publisher.delete_topic(topic_path) + publisher.delete_topic(request={"topic": topic_path}) @pytest.fixture(scope="module") @@ -100,13 +100,15 @@ def subscription_id(topic_id): topic_path = subscriber.topic_path(GCLOUD_PROJECT, topic_id) subscription_path = subscriber.subscription_path(GCLOUD_PROJECT, SUBSCRIPTION_ID) try: - subscriber.create_subscription(subscription_path, topic_path) + subscriber.create_subscription( + request={"name": subscription_path, "topic": topic_path} + ) except google.api_core.exceptions.AlreadyExists: pass yield SUBSCRIPTION_ID - subscriber.delete_subscription(subscription_path) + subscriber.delete_subscription(request={"subscription": subscription_path}) @pytest.fixture(scope="module") diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index b90eef00..bca0522e 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -17,6 +17,7 @@ import os from pathlib import Path import sys +from typing import Callable, Dict, List, Optional import nox @@ -68,7 +69,7 @@ TEST_CONFIG.update(TEST_CONFIG_OVERRIDE) -def get_pytest_env_vars(): +def get_pytest_env_vars() -> Dict[str, str]: """Returns a dict for pytest invocation.""" ret = {} @@ -97,7 +98,7 @@ def get_pytest_env_vars(): # -def _determine_local_import_names(start_dir): +def _determine_local_import_names(start_dir: str) -> List[str]: """Determines all import names that should be considered "local". This is used when running the linter to insure that import order is @@ -135,7 +136,7 @@ def _determine_local_import_names(start_dir): @nox.session -def lint(session): +def lint(session: nox.sessions.Session) -> None: if not TEST_CONFIG['enforce_type_hints']: session.install("flake8", "flake8-import-order") else: @@ -154,7 +155,7 @@ def lint(session): @nox.session -def blacken(session): +def blacken(session: nox.sessions.Session) -> None: session.install("black") python_files = [path for path in os.listdir(".") if path.endswith(".py")] @@ -168,7 +169,7 @@ def blacken(session): PYTEST_COMMON_ARGS = ["--junitxml=sponge_log.xml"] -def _session_tests(session, post_install=None): +def _session_tests(session: nox.sessions.Session, post_install: Callable = None) -> None: """Runs py.test for a particular project.""" if os.path.exists("requirements.txt"): session.install("-r", "requirements.txt") @@ -194,7 +195,7 @@ def _session_tests(session, post_install=None): @nox.session(python=ALL_VERSIONS) -def py(session): +def py(session: nox.sessions.Session) -> None: """Runs py.test for a sample using the specified version of Python.""" if session.python in TESTED_VERSIONS: _session_tests(session) @@ -209,7 +210,7 @@ def py(session): # -def _get_repo_root(): +def _get_repo_root() -> Optional[str]: """ Returns the root folder of the project. """ # Get root of this repository. Assume we don't have directories nested deeper than 10 items. p = Path(os.getcwd()) @@ -232,7 +233,7 @@ def _get_repo_root(): @nox.session @nox.parametrize("path", GENERATED_READMES) -def readmegen(session, path): +def readmegen(session: nox.sessions.Session, path: str) -> None: """(Re-)generates the readme for a sample.""" session.install("jinja2", "pyyaml") dir_ = os.path.dirname(path) diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index d1d72619..40f1fc59 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,5 +1,5 @@ -google-cloud-dlp==2.0.0 -google-cloud-storage==1.33.0 -google-cloud-pubsub==1.7.0 -google-cloud-datastore==1.15.3 -google-cloud-bigquery==2.4.0 +google-cloud-dlp==3.0.0 +google-cloud-storage==1.35.0 +google-cloud-pubsub==2.2.0 +google-cloud-datastore==2.1.0 +google-cloud-bigquery==2.7.0 diff --git a/samples/snippets/risk_test.py b/samples/snippets/risk_test.py index a790e127..ef2ac327 100644 --- a/samples/snippets/risk_test.py +++ b/samples/snippets/risk_test.py @@ -49,13 +49,13 @@ def topic_id(): publisher = google.cloud.pubsub.PublisherClient() topic_path = publisher.topic_path(GCLOUD_PROJECT, TOPIC_ID) try: - publisher.create_topic(topic_path) + publisher.create_topic(request={"name": topic_path}) except google.api_core.exceptions.AlreadyExists: pass yield TOPIC_ID - publisher.delete_topic(topic_path) + publisher.delete_topic(request={"topic": topic_path}) @pytest.fixture(scope="module") @@ -65,13 +65,15 @@ def subscription_id(topic_id): topic_path = subscriber.topic_path(GCLOUD_PROJECT, topic_id) subscription_path = subscriber.subscription_path(GCLOUD_PROJECT, SUBSCRIPTION_ID) try: - subscriber.create_subscription(subscription_path, topic_path) + subscriber.create_subscription( + request={"name": subscription_path, "topic": topic_path} + ) except google.api_core.exceptions.AlreadyExists: pass yield SUBSCRIPTION_ID - subscriber.delete_subscription(subscription_path) + subscriber.delete_subscription(request={"subscription": subscription_path}) @pytest.fixture(scope="module") @@ -118,12 +120,40 @@ def bigquery_project(): rows_to_insert = [(u"Gary Smith", u"My email is gary@example.com")] harmful_rows_to_insert = [ - (u"Gandalf", u"(123) 456-7890", "4231 5555 6781 9876", 27, "Male", "US",), - (u"Dumbledore", u"(313) 337-1337", "6291 8765 1095 7629", 27, "Male", "US",), + ( + u"Gandalf", + u"(123) 456-7890", + "4231 5555 6781 9876", + 27, + "Male", + "US", + ), + ( + u"Dumbledore", + u"(313) 337-1337", + "6291 8765 1095 7629", + 27, + "Male", + "US", + ), (u"Joe", u"(452) 123-1234", "3782 2288 1166 3030", 35, "Male", "US"), (u"James", u"(567) 890-1234", "8291 3627 8250 1234", 19, "Male", "US"), - (u"Marie", u"(452) 123-1234", "8291 3627 8250 1234", 35, "Female", "US",), - (u"Carrie", u"(567) 890-1234", "2253 5218 4251 4526", 35, "Female", "US",), + ( + u"Marie", + u"(452) 123-1234", + "8291 3627 8250 1234", + 35, + "Female", + "US", + ), + ( + u"Carrie", + u"(567) 890-1234", + "2253 5218 4251 4526", + 35, + "Female", + "US", + ), ] bigquery_client.insert_rows(table, rows_to_insert) diff --git a/setup.py b/setup.py index 660cc9f7..c2f605ee 100644 --- a/setup.py +++ b/setup.py @@ -21,7 +21,7 @@ name = "google-cloud-dlp" description = "Cloud Data Loss Prevention (DLP) API API client library" -version = "3.0.0" +version = "3.0.1" release_status = "Development Status :: 5 - Production/Stable" dependencies = [ "google-api-core[grpc] >= 1.22.0, < 2.0.0dev", diff --git a/synth.metadata b/synth.metadata index a5606163..69d5a27a 100644 --- a/synth.metadata +++ b/synth.metadata @@ -4,29 +4,29 @@ "git": { "name": ".", "remote": "https://github.com/googleapis/python-dlp.git", - "sha": "908a8d3e2ee741ebdbc78359a18c7e8a1255c5ea" + "sha": "1911aad2b399102a9a00bdcf676598c241729cfb" } }, { "git": { "name": "googleapis", "remote": "https://github.com/googleapis/googleapis.git", - "sha": "3f87da2ed1ddc3566ef0810c4fc06a2682cc9f5f", - "internalRef": "343022252" + "sha": "dd372aa22ded7a8ba6f0e03a80e06358a3fa0907", + "internalRef": "347055288" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "e89175cf074dccc4babb4eca66ae913696e47a71" + "sha": "373861061648b5fe5e0ac4f8a38b32d639ee93e4" } }, { "git": { "name": "synthtool", "remote": "https://github.com/googleapis/synthtool.git", - "sha": "e89175cf074dccc4babb4eca66ae913696e47a71" + "sha": "373861061648b5fe5e0ac4f8a38b32d639ee93e4" } } ], @@ -85,6 +85,7 @@ ".kokoro/test-samples.sh", ".kokoro/trampoline.sh", ".kokoro/trampoline_v2.sh", + ".pre-commit-config.yaml", ".trampolinerc", "CODE_OF_CONDUCT.md", "CONTRIBUTING.rst", @@ -119,7 +120,6 @@ "renovate.json", "samples/AUTHORING_GUIDE.md", "samples/CONTRIBUTING.md", - "samples/snippets/README.rst", "samples/snippets/noxfile.py", "scripts/decrypt-secrets.sh", "scripts/fixup_dlp_v2_keywords.py", diff --git a/tests/unit/gapic/dlp_v2/test_dlp_service.py b/tests/unit/gapic/dlp_v2/test_dlp_service.py index 7c5dd868..0f79c156 100644 --- a/tests/unit/gapic/dlp_v2/test_dlp_service.py +++ b/tests/unit/gapic/dlp_v2/test_dlp_service.py @@ -8023,6 +8023,10 @@ def test_dlp_service_transport_channel_mtls_with_client_cert_source(transport_cl scopes=("https://www.googleapis.com/auth/cloud-platform",), ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel assert transport._ssl_channel_credentials == mock_ssl_cred @@ -8061,6 +8065,10 @@ def test_dlp_service_transport_channel_mtls_with_adc(transport_class): scopes=("https://www.googleapis.com/auth/cloud-platform",), ssl_credentials=mock_ssl_cred, quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], ) assert transport.grpc_channel == mock_grpc_channel