diff --git a/packages/google-cloud-retail/samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md b/packages/google-cloud-retail/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md similarity index 51% rename from packages/google-cloud-retail/samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md rename to packages/google-cloud-retail/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md index 000f12d0e867..f7989d003e60 100644 --- a/packages/google-cloud-retail/samples/snippets/TEST_RESOURCES_SETUP_CLEANUP.md +++ b/packages/google-cloud-retail/samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md @@ -3,8 +3,10 @@ ## Required environment variables To successfully import the catalog data for tests, the following environment variables should be set: - - PROJECT_NUMBER + - GOOGLE_CLOUD_PROJECT_NUMBER + - GOOGLE_CLOUD_PROJECT_ID - BUCKET_NAME + - EVENTS_BUCKET_NAME These values are stored in the Secret Manager and will be submitted as docker environment variables before the test run. @@ -12,13 +14,20 @@ The Secret Manager name is set in .kokoro/presubmit/common.cfg file, SECRET_MANA ## Import catalog data -There is a JSON file with valid products prepared in the `product` directory: -`resources/products.json`. +There are JSON files with valid products and user events prepared in `resources` directory: +`samples/resources/products.json` and `samples/resources/user_events.json` respectively. Run the `create_test_resources.py` to perform the following actions: - create the GCS bucket , - - upload the product data from `resources/products.json` file, - - import products to the default branch of the Retail catalog. + - upload the product data from `resources/products.json` file to products bucket, + - import products to the default branch of the Retail catalog, + - create the GCS bucket , + - upload the product data from `resources/user_events.json` file to events bucket, + - create a BigQuery dataset and table `products`, + - insert products from resources/products.json to the created products table, + - create a BigQuery dataset and table `events`, + - insert user events from resources/user_events.json to the created events table + ``` $ python create_test_resources.py @@ -33,6 +42,10 @@ Run the `remove_test_resources.py` to perform the following actions: - remove all objects from the GCS bucket , - remove the bucket, - delete all products from the Retail catalog. + - remove all objects from the GCS bucket , + - remove the bucket, + - remove dataset `products` along with tables + - remove dataset `user_events` along with tables ``` $ python remove_test_resources.py diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/resources/events_schema.json b/packages/google-cloud-retail/samples/interactive-tutorials/resources/events_schema.json new file mode 100644 index 000000000000..a52c0e56f364 --- /dev/null +++ b/packages/google-cloud-retail/samples/interactive-tutorials/resources/events_schema.json @@ -0,0 +1,73 @@ +[ + { + "fields":[ + { + "mode": "NULLABLE", + "name": "currencyCode", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "revenue", + "type": "FLOAT" + } + ], + "mode": "NULLABLE", + "name": "purchaseTransaction", + "type": "RECORD" + }, + { + "fields":[ + { + "mode": "NULLABLE", + "name": "quantity", + "type": "INTEGER" + }, + { + "fields":[ + { + "mode": "NULLABLE", + "name": "id", + "type": "STRING" + } + ], + "mode": "NULLABLE", + "name": "product", + "type": "RECORD" + } + ], + "mode": "REPEATED", + "name": "productDetails", + "type": "RECORD" + }, + { + "mode": "REQUIRED", + "name": "eventTime", + "type": "STRING" + }, + { + "mode": "REQUIRED", + "name": "visitorId", + "type": "STRING" + }, + { + "mode": "REQUIRED", + "name": "eventType", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "searchQuery", + "type": "STRING" + }, + { + "mode": "NULLABLE", + "name": "cartId", + "type": "STRING" + }, + { + "mode": "REPEATED", + "name": "pageCategories", + "type": "STRING" + } + ] \ No newline at end of file diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/resources/product_schema.json b/packages/google-cloud-retail/samples/interactive-tutorials/resources/product_schema.json new file mode 100644 index 000000000000..2dcc79f7fe3c --- /dev/null +++ b/packages/google-cloud-retail/samples/interactive-tutorials/resources/product_schema.json @@ -0,0 +1,317 @@ +[ + { + "name": "name", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "id", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "primaryProductId", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "collectionMemberIds", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "gtin", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "categories", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "title", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "brands", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "description", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "languageCode", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "attributes", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "key", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "value", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "text", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "numbers", + "type": "FLOAT", + "mode": "REPEATED" + }, + { + "name": "searchable", + "type": "BOOLEAN", + "mode": "NULLABLE" + }, + { + "name": "indexable", + "type": "BOOLEAN", + "mode": "NULLABLE" + } + ] + } + ] + }, + { + "name": "tags", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "priceInfo", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "currencyCode", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "price", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "originalPrice", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "cost", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "priceEffectiveTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "priceExpireTime", + "type": "STRING", + "mode": "NULLABLE" + } + ] + }, + { + "name": "rating", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "ratingCount", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "averageRating", + "type": "FLOAT", + "mode": "NULLABLE" + }, + { + "name": "ratingHistogram", + "type": "INTEGER", + "mode": "REPEATED" + } + ] + }, + { + "name": "expireTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "ttl", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "seconds", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "nanos", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "availableTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "availability", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "availableQuantity", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "fulfillmentInfo", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "type", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "placeIds", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "uri", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "images", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "uri", + "type": "STRING", + "mode": "REQUIRED" + }, + { + "name": "height", + "type": "INTEGER", + "mode": "NULLABLE" + }, + { + "name": "width", + "type": "INTEGER", + "mode": "NULLABLE" + } + ] + }, + { + "name": "audience", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "genders", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "ageGroups", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "colorInfo", + "type": "RECORD", + "mode": "NULLABLE", + "fields": [ + { + "name": "colorFamilies", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "colors", + "type": "STRING", + "mode": "REPEATED" + } + ] + }, + { + "name": "sizes", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "materials", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "patterns", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "conditions", + "type": "STRING", + "mode": "REPEATED" + }, + { + "name": "retrievableFields", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "publishTime", + "type": "STRING", + "mode": "NULLABLE" + }, + { + "name": "promotions", + "type": "RECORD", + "mode": "REPEATED", + "fields": [ + { + "name": "promotionId", + "type": "STRING", + "mode": "NULLABLE" + } + ] + } +] \ No newline at end of file diff --git a/packages/google-cloud-retail/samples/snippets/resources/products.json b/packages/google-cloud-retail/samples/interactive-tutorials/resources/products.json similarity index 100% rename from packages/google-cloud-retail/samples/snippets/resources/products.json rename to packages/google-cloud-retail/samples/interactive-tutorials/resources/products.json diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/resources/products_some_invalid.json b/packages/google-cloud-retail/samples/interactive-tutorials/resources/products_some_invalid.json new file mode 100644 index 000000000000..f46dc76191c7 --- /dev/null +++ b/packages/google-cloud-retail/samples/interactive-tutorials/resources/products_some_invalid.json @@ -0,0 +1,3 @@ +{"id": "GGCOGOAC101259","name": "GGCOGOAC101259","title": "#IamRemarkable Pen","brands": ["#IamRemarkable"],"categories": ["Office"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "16"},"colorInfo": {"colorFamilies": ["Blue"],"colors": ["Light blue","Blue","Dark blue"]},"availability": "INVALID_VALUE","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGCOGOAC101259.jpg"}],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","attributes":[ {"key":"material", "value": {"indexable": "true","searchable": "true","text": ["Metal","Recycled Plastic"]}}],"uri": "https://shop.googlemerchandisestore.com/Google+Redesign/Office/IamRemarkable+Pen"} +{"id": "GGPRAHPL107110","name": "GGPRAHPL107110","title": "Android Iconic Hat Green","brands": ["Android"],"categories": ["Apparel"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "16"},"colorInfo": {"colorFamilies": ["Green"],"colors": ["Olive","Grass green","Light green"]},"availability": "IN_STOCK","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGOEAHPL130910.jpg"}],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","uri": "https://shop.googlemerchandisestore.com/Google+Prize+Portal/Android+Iconic+Hat+Green"} +{"id": "GGOEAAKQ137410","name": "GGOEAAKQ137410","title": "Android Iconic Sock","brands": ["Android"],"categories": ["Apparel"],"priceInfo": {"cost": "12.0","currencyCode": "USD","originalPrice": "45.0","priceEffectiveTime": "2020-08-01T12:00:00+00:00","priceExpireTime": "2120-08-01T12:00:00+00:00","price": "17"},"availability": "IN_STOCK","availableQuantity": 50,"availableTime": "2021-10-11T12:00:00+00:00","images": [{"height": "300","width": "400","uri": "https://shop.googlemerchandisestore.com/store/20160512512/assets/items/images/GGOEAAKQ137410.jpg"}],"sizes": ["XS","S","M","L","XL"],"retrievableFields": "name,title,brands,categories,priceInfo,colorInfo,availability,images,attributes.material,attributes.ecofriendly,attributes.style,attributes.collection,uri","attributes":[ {"key":"collection", "value": {"indexable": "true","numbers": [2022.0]}},{"key":"material", "value": {"indexable": "true","searchable": "true","text": ["Polyester","Membrane"]}},{"key":"ecofriendly", "value": {"indexable": "false","searchable": "false","text": ["Low-impact fabrics","recycled fabrics","recycled packaging","plastic-free packaging","ethically made"]}},{"key":"style", "value": {"indexable": "true","searchable": "true","text": ["Sport","Functional"]}}],"uri": "https://shop.googlemerchandisestore.com/Google+Redesign/Apparel/Android+Iconic+Sock"} \ No newline at end of file diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/resources/user_events.json b/packages/google-cloud-retail/samples/interactive-tutorials/resources/user_events.json new file mode 100644 index 000000000000..5360c3097000 --- /dev/null +++ b/packages/google-cloud-retail/samples/interactive-tutorials/resources/user_events.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/resources/user_events_some_invalid.json b/packages/google-cloud-retail/samples/interactive-tutorials/resources/user_events_some_invalid.json new file mode 100644 index 000000000000..c98b16996476 --- /dev/null +++ b/packages/google-cloud-retail/samples/interactive-tutorials/resources/user_events_some_invalid.json @@ -0,0 +1,4 @@ +{"eventType":"home-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00"} +{"eventType":"invalid","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"RockerJeans teenagers blue jeans"} +{"eventType":"search","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","searchQuery":"SocksUnlimited teenagers black socks"} +{"eventType":"detail-page-view","visitorId":"bjbs_group1_visitor1","eventTime":"2021-12-12T10:27:42+00:00","productDetails":{"product":{"id":"GGCOGAEC100616"},"quantity":3}} diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/search/noxfile_config.py b/packages/google-cloud-retail/samples/interactive-tutorials/search/noxfile_config.py index 8eb8d16fb241..401ff7f24f5c 100644 --- a/packages/google-cloud-retail/samples/interactive-tutorials/search/noxfile_config.py +++ b/packages/google-cloud-retail/samples/interactive-tutorials/search/noxfile_config.py @@ -28,7 +28,5 @@ # 'gcloud_project_env': 'BUILD_SPECIFIC_GCLOUD_PROJECT', # A dictionary you want to inject into your test. Don't put any # secrets here. These values will override predefined values. - "envs": { - "BUCKET_NAME": "retail-interactive-tutorials", - }, + "envs": {"BUCKET_NAME": "retail-interactive-tutorials",}, } diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/search/search_simple_query_test.py b/packages/google-cloud-retail/samples/interactive-tutorials/search/search_simple_query_test.py index 32749e2a85c4..fc6d0c692b68 100644 --- a/packages/google-cloud-retail/samples/interactive-tutorials/search/search_simple_query_test.py +++ b/packages/google-cloud-retail/samples/interactive-tutorials/search/search_simple_query_test.py @@ -21,9 +21,7 @@ def test_search_simple_query_pass(): - output = str( - subprocess.check_output("python search_simple_query.py", shell=True) - ) + output = str(subprocess.check_output("python search_simple_query.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/search/search_with_filtering_test.py b/packages/google-cloud-retail/samples/interactive-tutorials/search/search_with_filtering_test.py index 270ff1939b65..5b10ae34a4d2 100644 --- a/packages/google-cloud-retail/samples/interactive-tutorials/search/search_with_filtering_test.py +++ b/packages/google-cloud-retail/samples/interactive-tutorials/search/search_with_filtering_test.py @@ -21,9 +21,7 @@ def test_search_with_filtering_pass(): - output = str( - subprocess.check_output("python search_with_filtering.py", shell=True) - ) + output = str(subprocess.check_output("python search_with_filtering.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/packages/google-cloud-retail/samples/interactive-tutorials/search/search_with_ordering_test.py b/packages/google-cloud-retail/samples/interactive-tutorials/search/search_with_ordering_test.py index 0de2c9798430..ec8fcfb9cff1 100644 --- a/packages/google-cloud-retail/samples/interactive-tutorials/search/search_with_ordering_test.py +++ b/packages/google-cloud-retail/samples/interactive-tutorials/search/search_with_ordering_test.py @@ -21,9 +21,7 @@ def test_search_with_ordering_pass(): - output = str( - subprocess.check_output("python search_with_ordering.py", shell=True) - ) + output = str(subprocess.check_output("python search_with_ordering.py", shell=True)) assert re.match(".*search request.*", output) assert re.match(".*search response.*", output) diff --git a/packages/google-cloud-retail/samples/snippets/create_test_resources.py b/packages/google-cloud-retail/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py similarity index 51% rename from packages/google-cloud-retail/samples/snippets/create_test_resources.py rename to packages/google-cloud-retail/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py index e56dd575faad..4fa080b340ac 100644 --- a/packages/google-cloud-retail/samples/snippets/create_test_resources.py +++ b/packages/google-cloud-retail/samples/interactive-tutorials/test_resources_recovery/create_test_resources.py @@ -14,6 +14,8 @@ import os import re +import shlex +import subprocess import time from google.cloud.storage.bucket import Bucket @@ -23,14 +25,26 @@ ImportProductsRequest, ProductInputConfig from google.cloud.retail_v2 import ProductServiceClient -project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') -bucket_name = os.getenv('BUCKET_NAME') -storage_client = storage.Client() -resource_file = "resources/products.json" -object_name = re.search('resources/(.*?)$', resource_file).group(1) +project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] +products_bucket_name = os.environ['BUCKET_NAME'] +events_bucket_name = os.environ['EVENTS_BUCKET_NAME'] +project_id = os.environ["GOOGLE_CLOUD_PROJECT_ID"] + +product_resource_file = "../resources/products.json" +events_source_file = "../resources/user_events.json" + +product_dataset = "products" +product_table = "products" +product_schema = "resources/product_schema.json" +events_dataset = "user_events" +events_table = "events" +events_schema = "resources/events_schema.json" + +object_name = re.search('resources/(.*?)$', product_resource_file).group(1) default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number) +storage_client = storage.Client() def create_bucket(bucket_name: str) -> Bucket: """Create a new bucket in Cloud Storage""" @@ -65,14 +79,14 @@ def check_if_bucket_exists(new_bucket_name): def upload_data_to_bucket(bucket: Bucket): """Upload data to a GCS bucket""" blob = bucket.blob(object_name) - blob.upload_from_filename(resource_file) - print("Data from {} has being uploaded to {}".format(resource_file, + blob.upload_from_filename(product_resource_file) + print("Data from {} has being uploaded to {}".format(product_resource_file, bucket.name)) def get_import_products_gcs_request(): """Get import products from gcs request""" - gcs_bucket = "gs://{}".format(bucket_name) + gcs_bucket = "gs://{}".format(products_bucket_name) gcs_errors_bucket = "{}/error".format(gcs_bucket) gcs_source = GcsSource() @@ -121,7 +135,78 @@ def import_products_from_gcs(): "Wait 2 -5 minutes till products become indexed in the catalog,\ after that they will be available for search") +def create_bq_dataset(dataset_name): + """Create a BigQuery dataset""" + print("Creating dataset {}".format(dataset_name)) + if dataset_name not in list_bq_datasets(): + create_dataset_command = 'bq --location=US mk -d --default_table_expiration 3600 --description "This is my dataset." {}:{}'.format( + project_id, dataset_name) + output = subprocess.check_output(shlex.split(create_dataset_command)) + print(output) + print("dataset is created") + else: + print("dataset {} already exists".format(dataset_name)) + + +def list_bq_datasets(): + """List BigQuery datasets in the project""" + list_dataset_command = "bq ls --project_id {}".format(project_id) + list_output = subprocess.check_output(shlex.split(list_dataset_command)) + datasets = re.split(r'\W+', str(list_output)) + return datasets + + +def create_bq_table(dataset, table_name, schema): + """Create a BigQuery table""" + print("Creating BigQuery table {}".format(table_name)) + if table_name not in list_bq_tables(dataset): + create_table_command = "bq mk --table {}:{}.{} {}".format( + project_id, + dataset, + table_name, schema) + output = subprocess.check_output(shlex.split(create_table_command)) + print(output) + print("table is created") + else: + print("table {} already exists".format(table_name)) + -created_bucket = create_bucket(bucket_name) -upload_data_to_bucket(created_bucket) +def list_bq_tables(dataset): + """List BigQuery tables in the dataset""" + list_tables_command = "bq ls {}:{}".format(project_id, dataset) + tables = subprocess.check_output(shlex.split(list_tables_command)) + return str(tables) + + +def upload_data_to_bq_table(dataset, table_name, source, schema): + """Upload data to the table from specified source file""" + print("Uploading data form {} to the table {}.{}".format(source, dataset, + table_name)) + upload_data_command = "bq load --source_format=NEWLINE_DELIMITED_JSON {}:{}.{} {} {}".format( + project_id, dataset, table_name, source, schema) + output = subprocess.check_output(shlex.split(upload_data_command)) + print(output) + + +# Create a GCS bucket with products.json file +created_products_bucket = create_bucket(products_bucket_name) +upload_data_to_bucket(created_products_bucket) + +# Create a GCS bucket with user_events.json file +created_events_bucket = create_bucket(events_bucket_name) +upload_data_to_bucket(created_events_bucket) + +# Import prodcuts from the GCS bucket to the Retail catalog import_products_from_gcs() + +# Create a BigQuery table with products +create_bq_dataset(product_dataset) +create_bq_table(product_dataset, product_table, product_schema) +upload_data_to_bq_table(product_dataset, product_table, + product_resource_file, product_schema) + +# Create a BigQuery table with user events +create_bq_dataset(events_dataset) +create_bq_table(events_dataset, events_table, events_schema) +upload_data_to_bq_table(events_dataset, events_table, events_source_file, + events_schema) diff --git a/packages/google-cloud-retail/samples/snippets/remove_test_resources.py b/packages/google-cloud-retail/samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py similarity index 74% rename from packages/google-cloud-retail/samples/snippets/remove_test_resources.py rename to packages/google-cloud-retail/samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py index 268917057bfd..41981ba2c519 100644 --- a/packages/google-cloud-retail/samples/snippets/remove_test_resources.py +++ b/packages/google-cloud-retail/samples/interactive-tutorials/test_resources_recovery/remove_test_resources.py @@ -13,6 +13,8 @@ # limitations under the License. import os +import shlex +import subprocess from google.api_core.exceptions import PermissionDenied from google.cloud.storage.bucket import Bucket @@ -21,8 +23,13 @@ from google.cloud.retail import DeleteProductRequest, ListProductsRequest, \ ProductServiceClient -project_number = os.getenv('GOOGLE_CLOUD_PROJECT_NUMBER') -bucket_name = os.getenv('BUCKET_NAME') +project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"] +product_bucket_name = os.environ['BUCKET_NAME'] +events_bucket_name = os.environ['EVENTS_BUCKET_NAME'] +project_id = os.environ["GOOGLE_CLOUD_PROJECT_ID"] + +product_dataset = "products" +events_dataset = "user_events" default_catalog = "projects/{0}/locations/global/catalogs/default_catalog/branches/default_branch".format( project_number) @@ -30,7 +37,7 @@ storage_client = storage.Client() -def delete_bucket(): +def delete_bucket(bucket_name): """Delete bucket""" try: bucket = storage_client.get_bucket(bucket_name) @@ -52,6 +59,7 @@ def delete_object_from_bucket(bucket: Bucket): def delete_all_products(): """Delete all products in the catalog""" + print("Deleting all products, please wait") product_client = ProductServiceClient() list_request = ListProductsRequest() list_request.parent = default_catalog @@ -69,5 +77,14 @@ def delete_all_products(): print(f"{delete_count} products were deleted from {default_catalog}") -delete_bucket() +def delete_bq_dataset_with_tables(dataset): + """Delete a BigQuery dataset with all tables""" + delete_dataset_command = "bq rm -r -d -f {}".format(dataset) + output = subprocess.check_output(shlex.split(delete_dataset_command)) + print(output) + +delete_bucket(product_bucket_name) +delete_bucket(events_bucket_name) delete_all_products() +delete_bq_dataset_with_tables(product_dataset) +delete_bq_dataset_with_tables(events_dataset)