Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore(tests): fix async filter sample tests #976

Merged
merged 1 commit into from
Jun 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
130 changes: 92 additions & 38 deletions samples/snippets/filters/filter_snippets_async.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import datetime
from google.cloud.bigtable.data import Row
from google.cloud._helpers import _datetime_from_microseconds
from google.cloud.bigtable.data import Row


# [START bigtable_filters_limit_row_sample_asyncio]
async def filter_limit_row_sample(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(row_filter=row_filters.RowSampleFilter(0.75))

Expand All @@ -32,8 +34,11 @@ async def filter_limit_row_sample(project_id, instance_id, table_id):
# [END bigtable_filters_limit_row_sample_asyncio]
# [START bigtable_filters_limit_row_regex_asyncio]
async def filter_limit_row_regex(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.RowKeyRegexFilter(".*#20190501$".encode("utf-8"))
Expand All @@ -48,8 +53,11 @@ async def filter_limit_row_regex(project_id, instance_id, table_id):
# [END bigtable_filters_limit_row_regex_asyncio]
# [START bigtable_filters_limit_cells_per_col_asyncio]
async def filter_limit_cells_per_col(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(row_filter=row_filters.CellsColumnLimitFilter(2))

Expand All @@ -62,8 +70,11 @@ async def filter_limit_cells_per_col(project_id, instance_id, table_id):
# [END bigtable_filters_limit_cells_per_col_asyncio]
# [START bigtable_filters_limit_cells_per_row_asyncio]
async def filter_limit_cells_per_row(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(row_filter=row_filters.CellsRowLimitFilter(2))

Expand All @@ -76,8 +87,11 @@ async def filter_limit_cells_per_row(project_id, instance_id, table_id):
# [END bigtable_filters_limit_cells_per_row_asyncio]
# [START bigtable_filters_limit_cells_per_row_offset_asyncio]
async def filter_limit_cells_per_row_offset(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(row_filter=row_filters.CellsRowOffsetFilter(2))

Expand All @@ -90,8 +104,11 @@ async def filter_limit_cells_per_row_offset(project_id, instance_id, table_id):
# [END bigtable_filters_limit_cells_per_row_offset_asyncio]
# [START bigtable_filters_limit_col_family_regex_asyncio]
async def filter_limit_col_family_regex(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.FamilyNameRegexFilter("stats_.*$".encode("utf-8"))
Expand All @@ -106,8 +123,11 @@ async def filter_limit_col_family_regex(project_id, instance_id, table_id):
# [END bigtable_filters_limit_col_family_regex_asyncio]
# [START bigtable_filters_limit_col_qualifier_regex_asyncio]
async def filter_limit_col_qualifier_regex(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.ColumnQualifierRegexFilter(
Expand All @@ -124,8 +144,11 @@ async def filter_limit_col_qualifier_regex(project_id, instance_id, table_id):
# [END bigtable_filters_limit_col_qualifier_regex_asyncio]
# [START bigtable_filters_limit_col_range_asyncio]
async def filter_limit_col_range(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.ColumnRangeFilter(
Expand All @@ -142,8 +165,11 @@ async def filter_limit_col_range(project_id, instance_id, table_id):
# [END bigtable_filters_limit_col_range_asyncio]
# [START bigtable_filters_limit_value_range_asyncio]
async def filter_limit_value_range(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.ValueRangeFilter(b"PQ2A.190405", b"PQ2A.190406")
Expand All @@ -160,8 +186,11 @@ async def filter_limit_value_range(project_id, instance_id, table_id):


async def filter_limit_value_regex(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.ValueRegexFilter("PQ2A.*$".encode("utf-8"))
Expand All @@ -177,8 +206,12 @@ async def filter_limit_value_regex(project_id, instance_id, table_id):
# [START bigtable_filters_limit_timestamp_range_asyncio]
async def filter_limit_timestamp_range(project_id, instance_id, table_id):
import datetime
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters

from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

end = datetime.datetime(2019, 5, 1)

Expand All @@ -193,8 +226,11 @@ async def filter_limit_timestamp_range(project_id, instance_id, table_id):
# [END bigtable_filters_limit_timestamp_range_asyncio]
# [START bigtable_filters_limit_block_all_asyncio]
async def filter_limit_block_all(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(row_filter=row_filters.BlockAllFilter(True))

Expand All @@ -207,8 +243,11 @@ async def filter_limit_block_all(project_id, instance_id, table_id):
# [END bigtable_filters_limit_block_all_asyncio]
# [START bigtable_filters_limit_pass_all_asyncio]
async def filter_limit_pass_all(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(row_filter=row_filters.PassAllFilter(True))

Expand All @@ -221,8 +260,11 @@ async def filter_limit_pass_all(project_id, instance_id, table_id):
# [END bigtable_filters_limit_pass_all_asyncio]
# [START bigtable_filters_modify_strip_value_asyncio]
async def filter_modify_strip_value(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(row_filter=row_filters.StripValueTransformerFilter(True))

Expand All @@ -235,8 +277,11 @@ async def filter_modify_strip_value(project_id, instance_id, table_id):
# [END bigtable_filters_modify_strip_value_asyncio]
# [START bigtable_filters_modify_apply_label_asyncio]
async def filter_modify_apply_label(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(row_filter=row_filters.ApplyLabelFilter(label="labelled"))

Expand All @@ -249,8 +294,11 @@ async def filter_modify_apply_label(project_id, instance_id, table_id):
# [END bigtable_filters_modify_apply_label_asyncio]
# [START bigtable_filters_composing_chain_asyncio]
async def filter_composing_chain(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.RowFilterChain(
Expand All @@ -270,8 +318,11 @@ async def filter_composing_chain(project_id, instance_id, table_id):
# [END bigtable_filters_composing_chain_asyncio]
# [START bigtable_filters_composing_interleave_asyncio]
async def filter_composing_interleave(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.RowFilterUnion(
Expand All @@ -291,8 +342,11 @@ async def filter_composing_interleave(project_id, instance_id, table_id):
# [END bigtable_filters_composing_interleave_asyncio]
# [START bigtable_filters_composing_condition_asyncio]
async def filter_composing_condition(project_id, instance_id, table_id):
from google.cloud.bigtable.data import BigtableDataClientAsync, ReadRowsQuery
from google.cloud.bigtable.data import row_filters
from google.cloud.bigtable.data import (
BigtableDataClientAsync,
ReadRowsQuery,
row_filters,
)

query = ReadRowsQuery(
row_filter=row_filters.ConditionalRowFilter(
Expand Down
6 changes: 3 additions & 3 deletions samples/snippets/filters/filter_snippets_async_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@

import datetime
import os
import time

import inspect
from typing import AsyncGenerator
Expand All @@ -26,7 +25,6 @@
from . import filter_snippets_async
from google.cloud._helpers import (
_microseconds_from_datetime,
_datetime_from_microseconds,
)

PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
Expand Down Expand Up @@ -77,7 +75,7 @@ async def _populate_table(table_id):
timestamp = datetime.datetime(2019, 5, 1)
timestamp_minus_hr = timestamp - datetime.timedelta(hours=1)

async with (BigtableDataClientAsync(project=PROJECT) as client):
async with BigtableDataClientAsync(project=PROJECT) as client:
async with client.get_table(BIGTABLE_INSTANCE, table_id) as table:
async with table.mutations_batcher() as batcher:
await batcher.append(
Expand Down Expand Up @@ -257,6 +255,8 @@ async def _populate_table(table_id):

def _datetime_to_micros(value: datetime.datetime) -> int:
"""Uses the same conversion rules as the old client in"""
import calendar
import datetime as dt
if not value.tzinfo:
value = value.replace(tzinfo=datetime.timezone.utc)
# Regardless of what timezone is on the value, convert it to UTC.
Expand Down
5 changes: 2 additions & 3 deletions samples/snippets/filters/filters_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,16 @@


import datetime
import inspect
import os
import time
import uuid
import inspect

from google.cloud import bigtable
import pytest
from .snapshots.snap_filters_test import snapshots

from . import filter_snippets

from .snapshots.snap_filters_test import snapshots

PROJECT = os.environ["GOOGLE_CLOUD_PROJECT"]
BIGTABLE_INSTANCE = os.environ["BIGTABLE_INSTANCE"]
Expand Down
16 changes: 8 additions & 8 deletions samples/snippets/filters/noxfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@

import nox


# WARNING - WARNING - WARNING - WARNING - WARNING
# WARNING - WARNING - WARNING - WARNING - WARNING
# DO NOT EDIT THIS FILE EVER!
Expand Down Expand Up @@ -160,6 +159,7 @@ def blacken(session: nox.sessions.Session) -> None:
# format = isort + black
#


@nox.session
def format(session: nox.sessions.Session) -> None:
"""
Expand Down Expand Up @@ -187,7 +187,9 @@ def _session_tests(
session: nox.sessions.Session, post_install: Callable = None
) -> None:
# check for presence of tests
test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob("**/test_*.py", recursive=True)
test_list = glob.glob("**/*_test.py", recursive=True) + glob.glob(
"**/test_*.py", recursive=True
)
test_list.extend(glob.glob("**/tests", recursive=True))

if len(test_list) == 0:
Expand All @@ -209,9 +211,7 @@ def _session_tests(

if os.path.exists("requirements-test.txt"):
if os.path.exists("constraints-test.txt"):
session.install(
"-r", "requirements-test.txt", "-c", "constraints-test.txt"
)
session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt")
else:
session.install("-r", "requirements-test.txt")
with open("requirements-test.txt") as rtfile:
Expand All @@ -224,9 +224,9 @@ def _session_tests(
post_install(session)

if "pytest-parallel" in packages:
concurrent_args.extend(['--workers', 'auto', '--tests-per-worker', 'auto'])
concurrent_args.extend(["--workers", "auto", "--tests-per-worker", "auto"])
elif "pytest-xdist" in packages:
concurrent_args.extend(['-n', 'auto'])
concurrent_args.extend(["-n", "auto"])

session.run(
"pytest",
Expand Down Expand Up @@ -256,7 +256,7 @@ def py(session: nox.sessions.Session) -> None:


def _get_repo_root() -> Optional[str]:
""" Returns the root folder of the project. """
"""Returns the root folder of the project."""
# Get root of this repository. Assume we don't have directories nested deeper than 10 items.
p = Path(os.getcwd())
for i in range(10):
Expand Down
1 change: 1 addition & 0 deletions samples/snippets/filters/requirements-test.txt
Original file line number Diff line number Diff line change
@@ -1 +1,2 @@
pytest==7.4.4
pytest-asyncio
2 changes: 1 addition & 1 deletion samples/snippets/filters/requirements.txt
Original file line number Diff line number Diff line change
@@ -1 +1 @@
google-cloud-bigtable==2.22.0
google-cloud-bigtable==2.23.0
Loading