test: re-enable system_prerelease tests (#221)

* test: re-enable `system_prerelease` tests

* exclude ibis from prerelease install list

* install explicit 6.2.0 version for ibis in pre prelease

* add unit_prerelease to pre and post submit e2e tests

* Update noxfile.py

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Tim Swast <swast@google.com>
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>

.kokoro/continuous/e2e.cfg

@@ -3,5 +3,5 @@
 # Only run this nox session.
 env_vars: {
     key: "NOX_SESSION"
-    value: "system_noextras e2e notebook samples"
+    value: "unit_prerelease system_prerelease system_noextras e2e notebook samples"
 }

.kokoro/presubmit/e2e.cfg

@@ -3,5 +3,5 @@
 # Only run this nox session.
 env_vars: {
     key: "NOX_SESSION"
-    value: "system_noextras e2e notebook samples"
+    value: "unit_prerelease system_prerelease system_noextras e2e notebook samples"
 }

noxfile.py

@@ -494,6 +494,11 @@ def prerelease(session: nox.sessions.Session, tests_path):
         CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt"
     )
 
+    # Ignore officially released versions of certain packages specified in
+    # testing/constraints-*.txt and install a more recent, pre-release versions
+    # directly
+    already_installed = set()
+
     # PyArrow prerelease packages are published to an alternative PyPI host.
     # https://arrow.apache.org/docs/python/install.html#installing-nightly-packages
     session.install(
@@ -504,6 +509,8 @@ def prerelease(session: nox.sessions.Session, tests_path):
         "--upgrade",
         "pyarrow",
     )
+    already_installed.add("pyarrow")
+
     session.install(
         "--extra-index-url",
         "https://pypi.anaconda.org/scipy-wheels-nightly/simple",
@@ -512,16 +519,48 @@ def prerelease(session: nox.sessions.Session, tests_path):
         "--upgrade",
         "pandas",
     )
+    already_installed.add("pandas")
+
+    # TODO(shobs):
+    # Commit https://github.com/ibis-project/ibis/commit/c20ba7feab6bdea6c299721310e04dbc10551cc2
+    # introduced breaking change that removed the following:
+    #   ibis.expr.rules.column
+    #   ibis.expr.rules.value
+    #   ibis.expr.rules.any
+    # Let's exclude ibis head from prerelease install list for now. Instead, use
+    # a working ibis-framework version resolved via setup.by (currently resolves
+    # to version 6.2.0 due to version requirement "6.2.0,<7.0.0dev").
+    # We should enable the head back once bigframes support a version that
+    # includes the above commit.
+    # session.install(
+    #    "--upgrade",
+    #    "-e",  # Use -e so that py.typed file is included.
+    #    "git+https://github.com/ibis-project/ibis.git#egg=ibis-framework",
+    # )
+    session.install("--no-deps", "ibis-framework==6.2.0")
+    already_installed.add("ibis-framework")
+
+    # Workaround https://github.com/googleapis/python-db-dtypes-pandas/issues/178
+    session.install("--no-deps", "db-dtypes")
+    already_installed.add("db-dtypes")
+
+    # Ensure we catch breaking changes in the client libraries early.
+    session.install(
+        "--upgrade",
+        "-e",
+        "git+https://github.com/googleapis/python-bigquery.git#egg=google-cloud-bigquery",
+    )
+    already_installed.add("google-cloud-bigquery")
     session.install(
         "--upgrade",
-        "-e",  # Use -e so that py.typed file is included.
-        "git+https://github.com/ibis-project/ibis.git#egg=ibis-framework",
+        "-e",
+        "git+https://github.com/googleapis/python-bigquery-storage.git#egg=google-cloud-bigquery-storage",
     )
-    # Workaround https://github.com/googleapis/python-db-dtypes-pandas/issues/178
-    session.install("--no-deps", "db-dtypes")
+    already_installed.add("google-cloud-bigquery-storage")
 
     # Workaround to install pandas-gbq >=0.15.0, which is required by test only.
     session.install("--no-deps", "pandas-gbq")
+    already_installed.add("pandas-gbq")
 
     session.install(
         *set(UNIT_TEST_STANDARD_DEPENDENCIES + SYSTEM_TEST_STANDARD_DEPENDENCIES),
@@ -541,9 +580,6 @@ def prerelease(session: nox.sessions.Session, tests_path):
         constraints_text = constraints_file.read()
 
     # Ignore leading whitespace and comment lines.
-    already_installed = frozenset(
-        ("db-dtypes", "pandas", "pyarrow", "ibis-framework", "pandas-gbq")
-    )
     deps = [
         match.group(1)
         for match in re.finditer(