diff --git a/notebooks/getting_started/getting_started_bq_dataframes.ipynb b/notebooks/getting_started/getting_started_bq_dataframes.ipynb index 6cc6acc993..18be5e48fd 100644 --- a/notebooks/getting_started/getting_started_bq_dataframes.ipynb +++ b/notebooks/getting_started/getting_started_bq_dataframes.ipynb @@ -802,7 +802,7 @@ "source": [ "Running the cell below creates a custom function using the `remote_function` method. This function categorizes a value into one of two buckets: >= 4000 or <4000.\n", "\n", - "> Note: Creating a function requires a [BigQuery connection](https://cloud.google.com/bigquery/docs/remote-functions#create_a_remote_function). This code assumes a pre-created connection named `bigframes-rf-conn`. If\n", + "> Note: Creating a function requires a [BigQuery connection](https://cloud.google.com/bigquery/docs/remote-functions#create_a_remote_function). This code assumes a pre-created connection named `bigframes-default-connection`. If\n", "the connection is not already created, BigQuery DataFrames attempts to create one assuming the [necessary APIs\n", "and IAM permissions](https://cloud.google.com/python/docs/reference/bigframes/latest/bigframes.pandas#bigframes_pandas_remote_function) are set up in the project.\n", "\n", @@ -817,7 +817,7 @@ }, "outputs": [], "source": [ - "@bf.remote_function([float], str, bigquery_connection='bigframes-rf-conn')\n", + "@bf.remote_function([float], str)\n", "def get_bucket(num):\n", " if not num: return \"NA\"\n", " boundary = 4000\n", diff --git a/notebooks/remote_functions/remote_function.ipynb b/notebooks/remote_functions/remote_function.ipynb index 06be0e7293..063c1738b4 100644 --- a/notebooks/remote_functions/remote_function.ipynb +++ b/notebooks/remote_functions/remote_function.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 19, "id": "3613b1cd", "metadata": {}, "outputs": [], @@ -16,24 +16,16 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 20, "id": "f1175247", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shobs/code/bigframes1/venv/lib/python3.10/site-packages/google/auth/_default.py:78: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n", - " warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 25.4 s, sys: 2.5 s, total: 27.9 s\n", - "Wall time: 2min 31s\n" + "CPU times: user 2.34 s, sys: 307 ms, total: 2.65 s\n", + "Wall time: 17.8 s\n" ] }, { @@ -141,7 +133,7 @@ "9 154 Sure, but what about a solution using O(1) mem... 8" ] }, - "execution_count": 3, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -160,7 +152,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 21, "id": "fd8a04a3", "metadata": {}, "outputs": [], @@ -191,7 +183,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 22, "id": "2b5e4568", "metadata": {}, "outputs": [ @@ -199,8 +191,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 4.22 s, sys: 18.2 ms, total: 4.24 s\n", - "Wall time: 4.26 s\n" + "CPU times: user 3.32 s, sys: 0 ns, total: 3.32 s\n", + "Wall time: 3.32 s\n" ] }, { @@ -319,7 +311,7 @@ "9 154 Sure, but what about a solution using O(1) mem... 8 19" ] }, - "execution_count": 5, + "execution_count": 22, "metadata": {}, "output_type": "execute_result" } @@ -333,65 +325,25 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 23, "id": "b81feaef", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/google/home/shobs/code/bigframes1/venv/lib/python3.10/site-packages/google/auth/_default.py:78: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n", - " warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n", - "/usr/local/google/home/shobs/code/bigframes1/venv/lib/python3.10/site-packages/google/auth/_default.py:78: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n", - " warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n" - ] - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2b1c9d671db14d2ca3be6a0b0c698430", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HTML(value='Query job 6b0a39de-40a0-4dd4-be88-248bd8ebcd77 is RUNNING. " ] }, "metadata": {}, @@ -399,13 +351,11 @@ }, { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "09706700e8dd4cf39f65a0d58371c1eb", - "version_major": 2, - "version_minor": 0 - }, + "text/html": [ + "Query job a283cb39-41b1-44cd-a6c3-f2a2c6a55b25 is DONE. 17.2 GB processed. Open Job" + ], "text/plain": [ - "HTML(value='Query job 4c1d9d3e-be25-4818-b74d-6214164d99ab is DONE. 0 Bytes processed. " ] }, "metadata": {}, @@ -440,62 +390,62 @@ " \n", " \n", " 0\n", - " 11012908\n", - " you're welcome! according to the docs it shoul...\n", + " 11231597\n", + " In your update, why are some of the system fun...\n", " 0\n", " \n", " \n", " 1\n", - " 11013760\n", - " You *should* be concerned with the disk being ...\n", - " 0\n", + " 49684807\n", + " what you have tried so far . ??\n", + " 1\n", " \n", " \n", " 2\n", - " 11013784\n", - " have you looked at `Integrate` or `NIntegrate`?\n", + " 7623925\n", + " @Michael: It should work. Perhaps you looked i...\n", " 0\n", " \n", " \n", " 3\n", - " 11015512\n", - " sorry, is a typo. The variable name is dist. (...\n", + " 34046685\n", + " Will it work with SQL compact? Please excuse m...\n", " 0\n", " \n", " \n", " 4\n", - " 11016238\n", - " Pfff, I'm having trouble with that formula too...\n", + " 6426146\n", + " do you know the equation to your pdf?\n", " 0\n", " \n", " \n", " 5\n", - " 11016276\n", - " Thanks thinksteep! Does this mean that by usin...\n", + " 60686114\n", + " m sorry but at least you have to think about it.\n", " 0\n", " \n", " \n", " 6\n", - " 11016551\n", - " Jason, thanks for the reply. I've been workin...\n", + " 16631986\n", + " i think also making disable this by only jquer...\n", " 0\n", " \n", " \n", " 7\n", - " 11017973\n", - " I assume an `off` of 0.5 would put be exactly ...\n", + " 16498565\n", + " I am including these files on my header of the...\n", " 0\n", " \n", " \n", " 8\n", - " 11018225\n", - " Thank you very much. I do worry too much abou...\n", + " 26601001\n", + " wrong answer, you didn't understand the logic\n", " 0\n", " \n", " \n", " 9\n", - " 11018370\n", - " @IanClelland, I edited my question a bit. The ...\n", + " 73255842\n", + " Call the setOnClickListener before return row.\n", " 0\n", " \n", " \n", @@ -505,21 +455,21 @@ ], "text/plain": [ " id text score\n", - "0 11012908 you're welcome! according to the docs it shoul... 0\n", - "1 11013760 You *should* be concerned with the disk being ... 0\n", - "2 11013784 have you looked at `Integrate` or `NIntegrate`? 0\n", - "3 11015512 sorry, is a typo. The variable name is dist. (... 0\n", - "4 11016238 Pfff, I'm having trouble with that formula too... 0\n", - "5 11016276 Thanks thinksteep! Does this mean that by usin... 0\n", - "6 11016551 Jason, thanks for the reply. I've been workin... 0\n", - "7 11017973 I assume an `off` of 0.5 would put be exactly ... 0\n", - "8 11018225 Thank you very much. I do worry too much abou... 0\n", - "9 11018370 @IanClelland, I edited my question a bit. The ... 0\n", + "0 11231597 In your update, why are some of the system fun... 0\n", + "1 49684807 what you have tried so far . ?? 1\n", + "2 7623925 @Michael: It should work. Perhaps you looked i... 0\n", + "3 34046685 Will it work with SQL compact? Please excuse m... 0\n", + "4 6426146 do you know the equation to your pdf? 0\n", + "5 60686114 m sorry but at least you have to think about it. 0\n", + "6 16631986 i think also making disable this by only jquer... 0\n", + "7 16498565 I am including these files on my header of the... 0\n", + "8 26601001 wrong answer, you didn't understand the logic 0\n", + "9 73255842 Call the setOnClickListener before return row. 0\n", "\n", "[10 rows x 3 columns]" ] }, - "execution_count": 6, + "execution_count": 23, "metadata": {}, "output_type": "execute_result" } @@ -539,7 +489,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 24, "id": "55ed241e", "metadata": {}, "outputs": [ @@ -549,8 +499,9 @@ "text": [ "Help on function remote_function in module bigframes.pandas:\n", "\n", - "remote_function(input_types: 'List[type]', output_type: 'type', dataset: 'Optional[str]' = None, bigquery_connection: 'Optional[str]' = None, reuse: 'bool' = True)\n", - " Decorator to turn a user defined function into a BigQuery remote function.\n", + "remote_function(input_types: 'List[type]', output_type: 'type', dataset: 'Optional[str]' = None, bigquery_connection: 'Optional[str]' = None, reuse: 'bool' = True, name: 'Optional[str]' = None, packages: 'Optional[Sequence[str]]' = None)\n", + " Decorator to turn a user defined function into a BigQuery remote function. Check out\n", + " the code samples at: https://cloud.google.com/bigquery/docs/remote-functions#bigquery-dataframes.\n", " \n", " .. note::\n", " Please make sure following is setup before using this API:\n", @@ -576,7 +527,7 @@ " * BigQuery Data Editor (roles/bigquery.dataEditor)\n", " * BigQuery Connection Admin (roles/bigquery.connectionAdmin)\n", " * Cloud Functions Developer (roles/cloudfunctions.developer)\n", - " * Service Account User (roles/iam.serviceAccountUser)\n", + " * Service Account User (roles/iam.serviceAccountUser) on the service account `PROJECT_NUMBER-compute@developer.gserviceaccount.com`\n", " * Storage Object Viewer (roles/storage.objectViewer)\n", " * Project IAM Admin (roles/resourcemanager.projectIamAdmin) (Only required if the bigquery connection being used is not pre-created and is created dynamically with user credentials.)\n", " \n", @@ -602,15 +553,25 @@ " Name of the BigQuery connection. You should either have the\n", " connection already created in the `location` you have chosen, or\n", " you should have the Project IAM Admin role to enable the service\n", - " to create the connection for you if you need it.If this parameter is\n", + " to create the connection for you if you need it. If this parameter is\n", " not provided then the BigQuery connection from the session is used.\n", " reuse (bool, Optional):\n", " Reuse the remote function if already exists.\n", " `True` by default, which will result in reusing an existing remote\n", - " function (if any) that was previously created for the same udf.\n", - " Setting it to false would force creating a unique remote function.\n", + " function and corresponding cloud function (if any) that was\n", + " previously created for the same udf.\n", + " Setting it to `False` would force creating a unique remote function.\n", " If the required remote function does not exist then it would be\n", " created irrespective of this param.\n", + " name (str, Optional):\n", + " Explicit name of the persisted BigQuery remote function. Use it with\n", + " caution, because two users working in the same project and dataset\n", + " could overwrite each other's remote functions if they use the same\n", + " persistent name.\n", + " packages (str[], Optional):\n", + " Explicit name of the external package dependencies. Each dependency\n", + " is added to the `requirements.txt` as is, and can be of the form\n", + " supported in https://pip.pypa.io/en/stable/reference/requirements-file-format/.\n", " Returns:\n", " callable: A remote function object pointing to the cloud assets created\n", " in the background to support the remote execution. The cloud assets can be\n", @@ -631,49 +592,16 @@ }, { "cell_type": "code", - "execution_count": 8, - "id": "c9a8d03d", - "metadata": {}, - "outputs": [], - "source": [ - "# BigQuery DataFrames user is a data scientist and may not have privileges to\n", - "# create a BQ connector and set it up for invoking a cloud function. They\n", - "# should get such a connector created from their cloud admin and use it with\n", - "# BigQuery DataFrames remote functions. If the provided connection name does not\n", - "# exist, BigQuery DataFrames will try to create it on the fly assuming the user\n", - "# has sufficient privileges.\n", - "bq_connection_name = 'bigframes-rf-conn'" - ] - }, - { - "cell_type": "code", - "execution_count": 9, + "execution_count": 25, "id": "fbc27f81", "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[INFO][2023-08-18 21:23:29,687][bigframes.remote_function] Creating new cloud function: gcloud functions deploy bigframes-b0feb1fbaf8188b64d7e70118d93c5d4 --gen2 --runtime=python310 --project=bigframes-dev --region=us-central1 --source=/tmp/tmpl2ewfnue --entry-point=udf_http --trigger-http --no-allow-unauthenticated\n", - "[INFO][2023-08-18 21:24:43,689][bigframes.remote_function] Successfully created cloud function bigframes-b0feb1fbaf8188b64d7e70118d93c5d4 with uri (https://bigframes-b0feb1fbaf8188b64d7e70118d93c5d4-7krlje3eoq-uc.a.run.app)\n", - "[INFO][2023-08-18 21:24:57,348][bigframes.remote_function] Connector bigframes-rf-conn already exists\n", - "[INFO][2023-08-18 21:24:57,351][bigframes.remote_function] Creating BQ remote function: \n", - " CREATE OR REPLACE FUNCTION `bigframes-dev.bigframes_temp_us`.bigframes_b0feb1fbaf8188b64d7e70118d93c5d4(n INT64)\n", - " RETURNS INT64\n", - " REMOTE WITH CONNECTION `bigframes-dev.us.bigframes-rf-conn`\n", - " OPTIONS (\n", - " endpoint = \"https://bigframes-b0feb1fbaf8188b64d7e70118d93c5d4-7krlje3eoq-uc.a.run.app\"\n", - " )\n", - "[INFO][2023-08-18 21:24:58,300][bigframes.remote_function] Created remote function bigframes-dev.bigframes_temp_us.bigframes_b0feb1fbaf8188b64d7e70118d93c5d4\n" - ] - }, { "name": "stdout", "output_type": "stream", "text": [ "\n", - "Wall time: 89.0601 s\n" + "Wall time: 76.2628 s\n" ] } ], @@ -684,7 +612,7 @@ "\n", "# User defined function\n", "# https://www.codespeedy.com/find-nth-prime-number-in-python/\n", - "@pd.remote_function([int], int, bigquery_connection=bq_connection_name)\n", + "@pd.remote_function([int], int, reuse=False)\n", "def nth_prime(n):\n", " prime_numbers = [2,3]\n", " i=3\n", @@ -712,7 +640,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 26, "id": "c1c9355f", "metadata": {}, "outputs": [ @@ -720,33 +648,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 16.8 ms, sys: 61 µs, total: 16.8 ms\n", - "Wall time: 17 ms\n" + "CPU times: user 55.8 ms, sys: 182 µs, total: 56 ms\n", + "Wall time: 54.5 ms\n" ] }, { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "2f840ad27c514ed19c759a004b32de33", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HTML(value='Query job 0f421233-9d02-4746-bb39-86a3b0880aba is RUNNING. Open Job" + ], "text/plain": [ - "HTML(value='Query job 4f8d5734-8070-4630-8a59-c05a31d60476 is RUNNING. " ] }, "metadata": {}, @@ -754,13 +666,11 @@ }, { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "116d6ef3d6b247d3aaafef5fe6b970de", - "version_major": 2, - "version_minor": 0 - }, + "text/html": [ + "Query job c0a2c187-364d-4978-97bc-30352828f624 is DONE. 17.2 GB processed. Open Job" + ], "text/plain": [ - "HTML(value='Query job ec057f9e-726b-44f0-a5c0-24c05c7ecfeb is RUNNING. " ] }, "metadata": {}, @@ -796,71 +706,71 @@ " \n", " \n", " 0\n", - " 11012908\n", - " you're welcome! according to the docs it shoul...\n", + " 11231597\n", + " In your update, why are some of the system fun...\n", " 0\n", " -1\n", " \n", " \n", " 1\n", - " 11013760\n", - " You *should* be concerned with the disk being ...\n", - " 0\n", - " -1\n", + " 49684807\n", + " what you have tried so far . ??\n", + " 1\n", + " 2\n", " \n", " \n", " 2\n", - " 11013784\n", - " have you looked at `Integrate` or `NIntegrate`?\n", + " 7623925\n", + " @Michael: It should work. Perhaps you looked i...\n", " 0\n", " -1\n", " \n", " \n", " 3\n", - " 11015512\n", - " sorry, is a typo. The variable name is dist. (...\n", + " 34046685\n", + " Will it work with SQL compact? Please excuse m...\n", " 0\n", " -1\n", " \n", " \n", " 4\n", - " 11016238\n", - " Pfff, I'm having trouble with that formula too...\n", + " 6426146\n", + " do you know the equation to your pdf?\n", " 0\n", " -1\n", " \n", " \n", " 5\n", - " 11016276\n", - " Thanks thinksteep! Does this mean that by usin...\n", + " 60686114\n", + " m sorry but at least you have to think about it.\n", " 0\n", " -1\n", " \n", " \n", " 6\n", - " 11016551\n", - " Jason, thanks for the reply. I've been workin...\n", + " 16631986\n", + " i think also making disable this by only jquer...\n", " 0\n", " -1\n", " \n", " \n", " 7\n", - " 11017973\n", - " I assume an `off` of 0.5 would put be exactly ...\n", + " 16498565\n", + " I am including these files on my header of the...\n", " 0\n", " -1\n", " \n", " \n", " 8\n", - " 11018225\n", - " Thank you very much. I do worry too much abou...\n", + " 26601001\n", + " wrong answer, you didn't understand the logic\n", " 0\n", " -1\n", " \n", " \n", " 9\n", - " 11018370\n", - " @IanClelland, I edited my question a bit. The ...\n", + " 73255842\n", + " Call the setOnClickListener before return row.\n", " 0\n", " -1\n", " \n", @@ -871,21 +781,21 @@ ], "text/plain": [ " id text score n_prime\n", - "0 11012908 you're welcome! according to the docs it shoul... 0 -1\n", - "1 11013760 You *should* be concerned with the disk being ... 0 -1\n", - "2 11013784 have you looked at `Integrate` or `NIntegrate`? 0 -1\n", - "3 11015512 sorry, is a typo. The variable name is dist. (... 0 -1\n", - "4 11016238 Pfff, I'm having trouble with that formula too... 0 -1\n", - "5 11016276 Thanks thinksteep! Does this mean that by usin... 0 -1\n", - "6 11016551 Jason, thanks for the reply. I've been workin... 0 -1\n", - "7 11017973 I assume an `off` of 0.5 would put be exactly ... 0 -1\n", - "8 11018225 Thank you very much. I do worry too much abou... 0 -1\n", - "9 11018370 @IanClelland, I edited my question a bit. The ... 0 -1\n", + "0 11231597 In your update, why are some of the system fun... 0 -1\n", + "1 49684807 what you have tried so far . ?? 1 2\n", + "2 7623925 @Michael: It should work. Perhaps you looked i... 0 -1\n", + "3 34046685 Will it work with SQL compact? Please excuse m... 0 -1\n", + "4 6426146 do you know the equation to your pdf? 0 -1\n", + "5 60686114 m sorry but at least you have to think about it. 0 -1\n", + "6 16631986 i think also making disable this by only jquer... 0 -1\n", + "7 16498565 I am including these files on my header of the... 0 -1\n", + "8 26601001 wrong answer, you didn't understand the logic 0 -1\n", + "9 73255842 Call the setOnClickListener before return row. 0 -1\n", "\n", "[10 rows x 4 columns]" ] }, - "execution_count": 10, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -900,7 +810,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 27, "id": "2701cb81", "metadata": {}, "outputs": [ @@ -908,8 +818,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "bigframes-dev.bigframes_temp_us.bigframes_b0feb1fbaf8188b64d7e70118d93c5d4\n", - "projects/bigframes-dev/locations/us-central1/functions/bigframes-b0feb1fbaf8188b64d7e70118d93c5d4\n" + "shobs-test.bigframes_temp_us.bigframes_343b7b4bb93ca8747dae20c22bdaec8b_p27heyce\n", + "projects/shobs-test/locations/us-central1/functions/bigframes-343b7b4bb93ca8747dae20c22bdaec8b-p27heyce\n" ] } ], @@ -922,7 +832,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 28, "id": "920fa18e", "metadata": {}, "outputs": [ @@ -937,6 +847,42 @@ " \n", " Then it can be applied to a DataFrame or Series.\n", " \n", + " .. note::\n", + " The return type of the function must be explicitly specified in the\n", + " function's original definition even if not otherwise required.\n", + " \n", + " BigQuery Utils provides many public functions under the ``bqutil`` project on Google Cloud Platform project\n", + " (See: https://github.com/GoogleCloudPlatform/bigquery-utils/tree/master/udfs#using-the-udfs).\n", + " You can checkout Community UDFs to use community-contributed functions.\n", + " (See: https://github.com/GoogleCloudPlatform/bigquery-utils/tree/master/udfs/community#community-udfs).\n", + " \n", + " **Examples:**\n", + " \n", + " Use the ``cw_lower_case_ascii_only`` function from Community UDFs.\n", + " (https://github.com/GoogleCloudPlatform/bigquery-utils/blob/master/udfs/community/cw_lower_case_ascii_only.sqlx)\n", + " \n", + " >>> import bigframes.pandas as bpd\n", + " >>> bpd.options.display.progress_bar = None\n", + " \n", + " >>> df = bpd.DataFrame({'id': [1, 2, 3], 'name': ['AURÉLIE', 'CÉLESTINE', 'DAPHNÉ']})\n", + " >>> df\n", + " id name\n", + " 0 1 AURÉLIE\n", + " 1 2 CÉLESTINE\n", + " 2 3 DAPHNÉ\n", + " \n", + " [3 rows x 2 columns]\n", + " \n", + " >>> func = bpd.read_gbq_function(\"bqutil.fn.cw_lower_case_ascii_only\")\n", + " >>> df1 = df.assign(new_name=df['name'].apply(func))\n", + " >>> df1\n", + " id name new_name\n", + " 0 1 AURÉLIE aurÉlie\n", + " 1 2 CÉLESTINE cÉlestine\n", + " 2 3 DAPHNÉ daphnÉ\n", + " \n", + " [3 rows x 3 columns]\n", + " \n", " Args:\n", " function_name (str):\n", " the function's name in BigQuery in the format\n", @@ -965,7 +911,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 29, "id": "a6c9da0a", "metadata": {}, "outputs": [], @@ -978,7 +924,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 30, "id": "d7e7de7f", "metadata": {}, "outputs": [ @@ -986,33 +932,17 @@ "name": "stdout", "output_type": "stream", "text": [ - "CPU times: user 10.9 ms, sys: 0 ns, total: 10.9 ms\n", - "Wall time: 11.4 ms\n" + "CPU times: user 70.8 ms, sys: 3.49 ms, total: 74.3 ms\n", + "Wall time: 75.2 ms\n" ] }, { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "73d1a73593cb4115821ab128c221a48d", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HTML(value='Query job bec5f7d1-3df1-4292-8c68-c396bce7dc5d is RUNNING. Open Job" + ], "text/plain": [ - "HTML(value='Query job 02e3bf43-a387-41c7-85c7-4a5366251de7 is RUNNING. " ] }, "metadata": {}, @@ -1020,13 +950,11 @@ }, { "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "faf93766ce1e489183c86a9daf5ce7d1", - "version_major": 2, - "version_minor": 0 - }, + "text/html": [ + "Query job 4d3da7ed-42e6-4b2b-b656-ac9ef6d2e871 is DONE. 17.2 GB processed. Open Job" + ], "text/plain": [ - "HTML(value='Query job fa4329e8-2918-44c4-96c5-d8591364abc9 is RUNNING. " ] }, "metadata": {}, @@ -1063,80 +991,80 @@ " \n", " \n", " 0\n", - " 11012908\n", - " you're welcome! according to the docs it shoul...\n", + " 11231597\n", + " In your update, why are some of the system fun...\n", " 0\n", " -1\n", " -1\n", " \n", " \n", " 1\n", - " 11013760\n", - " You *should* be concerned with the disk being ...\n", - " 0\n", - " -1\n", - " -1\n", + " 49684807\n", + " what you have tried so far . ??\n", + " 1\n", + " 2\n", + " 2\n", " \n", " \n", " 2\n", - " 11013784\n", - " have you looked at `Integrate` or `NIntegrate`?\n", + " 7623925\n", + " @Michael: It should work. Perhaps you looked i...\n", " 0\n", " -1\n", " -1\n", " \n", " \n", " 3\n", - " 11015512\n", - " sorry, is a typo. The variable name is dist. (...\n", + " 34046685\n", + " Will it work with SQL compact? Please excuse m...\n", " 0\n", " -1\n", " -1\n", " \n", " \n", " 4\n", - " 11016238\n", - " Pfff, I'm having trouble with that formula too...\n", + " 6426146\n", + " do you know the equation to your pdf?\n", " 0\n", " -1\n", " -1\n", " \n", " \n", " 5\n", - " 11016276\n", - " Thanks thinksteep! Does this mean that by usin...\n", + " 60686114\n", + " m sorry but at least you have to think about it.\n", " 0\n", " -1\n", " -1\n", " \n", " \n", " 6\n", - " 11016551\n", - " Jason, thanks for the reply. I've been workin...\n", + " 16631986\n", + " i think also making disable this by only jquer...\n", " 0\n", " -1\n", " -1\n", " \n", " \n", " 7\n", - " 11017973\n", - " I assume an `off` of 0.5 would put be exactly ...\n", + " 16498565\n", + " I am including these files on my header of the...\n", " 0\n", " -1\n", " -1\n", " \n", " \n", " 8\n", - " 11018225\n", - " Thank you very much. I do worry too much abou...\n", + " 26601001\n", + " wrong answer, you didn't understand the logic\n", " 0\n", " -1\n", " -1\n", " \n", " \n", " 9\n", - " 11018370\n", - " @IanClelland, I edited my question a bit. The ...\n", + " 73255842\n", + " Call the setOnClickListener before return row.\n", " 0\n", " -1\n", " -1\n", @@ -1148,20 +1076,20 @@ ], "text/plain": [ " id text score \\\n", - "0 11012908 you're welcome! according to the docs it shoul... 0 \n", - "1 11013760 You *should* be concerned with the disk being ... 0 \n", - "2 11013784 have you looked at `Integrate` or `NIntegrate`? 0 \n", - "3 11015512 sorry, is a typo. The variable name is dist. (... 0 \n", - "4 11016238 Pfff, I'm having trouble with that formula too... 0 \n", - "5 11016276 Thanks thinksteep! Does this mean that by usin... 0 \n", - "6 11016551 Jason, thanks for the reply. I've been workin... 0 \n", - "7 11017973 I assume an `off` of 0.5 would put be exactly ... 0 \n", - "8 11018225 Thank you very much. I do worry too much abou... 0 \n", - "9 11018370 @IanClelland, I edited my question a bit. The ... 0 \n", + "0 11231597 In your update, why are some of the system fun... 0 \n", + "1 49684807 what you have tried so far . ?? 1 \n", + "2 7623925 @Michael: It should work. Perhaps you looked i... 0 \n", + "3 34046685 Will it work with SQL compact? Please excuse m... 0 \n", + "4 6426146 do you know the equation to your pdf? 0 \n", + "5 60686114 m sorry but at least you have to think about it. 0 \n", + "6 16631986 i think also making disable this by only jquer... 0 \n", + "7 16498565 I am including these files on my header of the... 0 \n", + "8 26601001 wrong answer, you didn't understand the logic 0 \n", + "9 73255842 Call the setOnClickListener before return row. 0 \n", "\n", " n_prime n_prime_again \n", "0 -1 -1 \n", - "1 -1 -1 \n", + "1 2 2 \n", "2 -1 -1 \n", "3 -1 -1 \n", "4 -1 -1 \n", @@ -1174,7 +1102,7 @@ "[10 rows x 5 columns]" ] }, - "execution_count": 15, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -1186,6 +1114,38 @@ "df = df.assign(n_prime_again=df['score'].apply(nth_prime_existing))\n", "df.head(10)" ] + }, + { + "cell_type": "code", + "execution_count": 31, + "id": "bafab950", + "metadata": {}, + "outputs": [], + "source": [ + "# Clean up GCP assets created as part of bigframes remote_function\n", + "def cleanup_remote_function_assets(remote_udf, ignore_failures=False):\n", + " \"\"\"Clean up the GCP assets behind a bigframes remote function.\"\"\"\n", + "\n", + " session = pd.get_global_session()\n", + "\n", + " # Clean up BQ remote function\n", + " try:\n", + " session.bqclient.delete_routine(remote_udf.bigframes_remote_function)\n", + " except Exception:\n", + " # By default don't raise exception in cleanup\n", + " if not ignore_failures:\n", + " raise\n", + "\n", + " # Clean up cloud function\n", + " try:\n", + " session.cloudfunctionsclient.delete_function(name=remote_udf.bigframes_cloud_function)\n", + " except Exception:\n", + " # By default don't raise exception in cleanup\n", + " if not ignore_failures:\n", + " raise\n", + "\n", + "cleanup_remote_function_assets(nth_prime)" + ] } ], "metadata": { diff --git a/samples/snippets/remote_function.py b/samples/snippets/remote_function.py index 646d7b0c30..61b7dc092a 100644 --- a/samples/snippets/remote_function.py +++ b/samples/snippets/remote_function.py @@ -38,8 +38,8 @@ def run_remote_function_and_read_gbq_function(project_id: str): # function. It requires a BigQuery connection. If the connection is not # already created, BigQuery DataFrames will attempt to create one assuming # the necessary APIs and IAM permissions are setup in the project. In our - # examples we would be using a pre-created connection named - # `bigframes-rf-conn`. We will also set `reuse=False` to make sure we don't + # examples we will be letting the default connection `bigframes-default-connection` + # be used. We will also set `reuse=False` to make sure we don't # step over someone else creating remote function in the same project from # the exact same source code at the same time. Let's try a `pandas`-like use # case in which we want to apply a user defined scalar function to every @@ -49,7 +49,6 @@ def run_remote_function_and_read_gbq_function(project_id: str): @bpd.remote_function( [float], str, - bigquery_connection="bigframes-rf-conn", reuse=False, ) def get_bucket(num): @@ -94,7 +93,6 @@ def get_bucket(num): @bpd.remote_function( [str], str, - bigquery_connection="bigframes-rf-conn", reuse=False, packages=["cryptography"], )