Skip to content

Commit

Permalink
feat: Publish GenAI Evaluation SDK GA features to `vertexai.evaluatio…
Browse files Browse the repository at this point in the history
…n` module. Switch GenAI Evaluation Service client to v1 version.

PiperOrigin-RevId: 665153547
  • Loading branch information
jsondai authored and copybara-github committed Aug 20, 2024
1 parent 9f1e073 commit 45e4251
Show file tree
Hide file tree
Showing 19 changed files with 160 additions and 65 deletions.
20 changes: 10 additions & 10 deletions tests/unit/vertexai/test_evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -24,23 +24,23 @@
import vertexai
from google.cloud.aiplatform import initializer
from google.cloud.aiplatform.metadata import metadata
from google.cloud.aiplatform_v1beta1.services import (
from google.cloud.aiplatform_v1.services import (
evaluation_service as gapic_evaluation_services,
)
from google.cloud.aiplatform_v1beta1.types import (
from google.cloud.aiplatform_v1.types import (
evaluation_service as gapic_evaluation_service_types,
)
from vertexai import evaluation
from vertexai import generative_models
from vertexai.preview import evaluation
from vertexai.preview.evaluation import _base as eval_base
from vertexai.preview.evaluation import _evaluation
from vertexai.preview.evaluation import utils
from vertexai.preview.evaluation.metrics import _rouge
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation import _base as eval_base
from vertexai.evaluation import _evaluation
from vertexai.evaluation import utils
from vertexai.evaluation.metrics import _rouge
from vertexai.evaluation.metrics import (
metric_prompt_template_examples,
)
from vertexai.preview.evaluation.metrics import pairwise_metric
from vertexai.preview.evaluation.metrics import pointwise_metric
from vertexai.evaluation.metrics import pairwise_metric
from vertexai.evaluation.metrics import pointwise_metric
import numpy as np
import pandas as pd
import pytest
Expand Down
45 changes: 45 additions & 0 deletions vertexai/evaluation/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# -*- coding: utf-8 -*-

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Vertex Gen AI Evaluation Service Module."""

from vertexai.evaluation import _base
from vertexai.evaluation import eval_task
from vertexai.evaluation import metrics
from vertexai.evaluation import prompt_template


EvalResult = _base.EvalResult
EvalTask = eval_task.EvalTask
PairwiseMetric = metrics.PairwiseMetric
PointwiseMetric = metrics.PointwiseMetric
CustomMetric = metrics.CustomMetric
PromptTemplate = prompt_template.PromptTemplate
PairwiseMetricPromptTemplate = metrics.PairwiseMetricPromptTemplate
PointwiseMetricPromptTemplate = metrics.PointwiseMetricPromptTemplate
MetricPromptTemplateExamples = metrics.MetricPromptTemplateExamples

__all__ = [
"EvalTask",
"EvalResult",
"PairwiseMetric",
"PointwiseMetric",
"CustomMetric",
"PromptTemplate",
"PairwiseMetricPromptTemplate",
"PointwiseMetricPromptTemplate",
"MetricPromptTemplateExamples",
]
Original file line number Diff line number Diff line change
Expand Up @@ -20,10 +20,10 @@
import dataclasses
from typing import Dict, List, Optional, Union, TYPE_CHECKING

from google.cloud.aiplatform_v1beta1.services import (
from google.cloud.aiplatform_v1.services import (
evaluation_service as gapic_evaluation_services,
)
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import (
_base as metrics_base,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,23 +28,23 @@
content as gapic_content_types,
)
from vertexai import generative_models
from vertexai.preview.evaluation import _base as evaluation_base
from vertexai.preview.evaluation import constants
from vertexai.preview.evaluation import (
from vertexai.evaluation import _base as evaluation_base
from vertexai.evaluation import constants
from vertexai.evaluation import (
prompt_template as prompt_template_base,
)
from vertexai.preview.evaluation import utils
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation import utils
from vertexai.evaluation.metrics import (
_base as metrics_base,
)
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import (
_instance_evaluation,
)
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import (
metric_prompt_template_examples,
)
from vertexai.preview.evaluation.metrics import pairwise_metric
from vertexai.preview.evaluation.metrics import pointwise_metric
from vertexai.evaluation.metrics import pairwise_metric
from vertexai.evaluation.metrics import pointwise_metric


try:
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -22,17 +22,17 @@
from google.cloud.aiplatform import base
from google.cloud.aiplatform.metadata import metadata
from vertexai import generative_models
from vertexai.preview.evaluation import _base as eval_base
from vertexai.preview.evaluation import _evaluation
from vertexai.preview.evaluation import constants
from vertexai.preview.evaluation import utils
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation import _base as eval_base
from vertexai.evaluation import _evaluation
from vertexai.evaluation import constants
from vertexai.evaluation import utils
from vertexai.evaluation.metrics import (
_base as metrics_base,
)
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import (
pairwise_metric,
)
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import (
pointwise_metric,
)
import numpy as np
Expand Down
50 changes: 50 additions & 0 deletions vertexai/evaluation/metrics/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# -*- coding: utf-8 -*-

# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Evaluation Metrics Module."""

from vertexai.evaluation.metrics import _base
from vertexai.evaluation.metrics import _rouge
from vertexai.evaluation.metrics import (
metric_prompt_template,
)
from vertexai.evaluation.metrics import (
metric_prompt_template_examples,
)
from vertexai.evaluation.metrics import pairwise_metric
from vertexai.evaluation.metrics import pointwise_metric


PairwiseMetric = pairwise_metric.PairwiseMetric
PointwiseMetric = pointwise_metric.PointwiseMetric
CustomMetric = _base.CustomMetric
PairwiseMetricPromptTemplate = metric_prompt_template.PairwiseMetricPromptTemplate
PointwiseMetricPromptTemplate = metric_prompt_template.PointwiseMetricPromptTemplate
MetricPromptTemplateExamples = (
metric_prompt_template_examples.MetricPromptTemplateExamples
)
Rouge = _rouge.Rouge


__all__ = [
"CustomMetric",
"PairwiseMetric",
"PointwiseMetric",
"PairwiseMetricPromptTemplate",
"PointwiseMetricPromptTemplate",
"MetricPromptTemplateExamples",
"Rouge",
]
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
import abc
from typing import Any, Callable, Dict, Literal, Union

from vertexai.preview.evaluation import constants
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation import constants
from vertexai.evaluation.metrics import (
metric_prompt_template as metric_prompt_template_base,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,24 @@
from google import api_core
from google.cloud.aiplatform import base
from google.cloud.aiplatform import initializer
from google.cloud.aiplatform_v1beta1.services import (
from google.cloud.aiplatform_v1.services import (
evaluation_service as gapic_evaluation_services,
)
from google.cloud.aiplatform_v1beta1.types import (
from google.cloud.aiplatform_v1.types import (
evaluation_service as gapic_eval_service_types,
)
from vertexai.preview.evaluation import _base as eval_base
from vertexai.preview.evaluation import constants
from vertexai.preview.evaluation import utils
from vertexai.preview.evaluation import (
from vertexai.evaluation import _base as eval_base
from vertexai.evaluation import constants
from vertexai.evaluation import utils
from vertexai.evaluation import (
prompt_template as prompt_template_base,
)
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import (
_base as metrics_base,
)
from vertexai.preview.evaluation.metrics import _rouge
from vertexai.preview.evaluation.metrics import pairwise_metric
from vertexai.preview.evaluation.metrics import pointwise_metric
from vertexai.evaluation.metrics import _rouge
from vertexai.evaluation.metrics import pairwise_metric
from vertexai.evaluation.metrics import pointwise_metric
from google.protobuf import json_format


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@
# limitations under the License.
#
from typing import Literal
from vertexai.preview.evaluation import constants
from vertexai.preview.evaluation.metrics import _base
from vertexai.evaluation import constants
from vertexai.evaluation.metrics import _base


class Rouge(_base._AutomaticMetric): # pylint: disable=protected-access
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
from typing import Dict, List, Optional

from google.cloud.aiplatform import base
from vertexai.preview.evaluation import (
from vertexai.evaluation import (
prompt_template,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,12 @@

from typing import List

from vertexai.preview.evaluation import constants
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation import constants
from vertexai.evaluation.metrics import (
_default_templates,
)
from vertexai.preview.evaluation.metrics import pairwise_metric
from vertexai.preview.evaluation.metrics import pointwise_metric
from vertexai.evaluation.metrics import pairwise_metric
from vertexai.evaluation.metrics import pointwise_metric
import immutabledict


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@
from typing import Callable, Optional, Union

from vertexai import generative_models
from vertexai.preview.evaluation.metrics import _base
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import _base
from vertexai.evaluation.metrics import (
metric_prompt_template as metric_prompt_template_base,
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@

from typing import Union

from vertexai.preview.evaluation.metrics import _base
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import _base
from vertexai.evaluation.metrics import (
metric_prompt_template as metric_prompt_template_base,
)

Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,10 @@
from google.cloud.aiplatform import compat
from google.cloud.aiplatform import initializer
from google.cloud.aiplatform import utils
from google.cloud.aiplatform_v1beta1.services import (
from google.cloud.aiplatform_v1.services import (
evaluation_service as gapic_evaluation_services,
)
from vertexai.preview.evaluation import constants
from vertexai.evaluation import constants


if TYPE_CHECKING:
Expand All @@ -44,10 +44,10 @@

class _EvaluationServiceClientWithOverride(utils.ClientWithOverride):
_is_temporary = False
_default_version = compat.V1BETA1
_default_version = compat.V1
_version_map = (
(
compat.V1BETA1,
compat.V1,
gapic_evaluation_services.EvaluationServiceClient,
),
)
Expand Down
8 changes: 4 additions & 4 deletions vertexai/preview/evaluation/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,10 @@
#
"""Vertex Gen AI Evaluation Service Module."""

from vertexai.preview.evaluation import _base
from vertexai.preview.evaluation import eval_task
from vertexai.preview.evaluation import metrics
from vertexai.preview.evaluation import prompt_template
from vertexai.evaluation import _base
from vertexai.evaluation import eval_task
from vertexai.evaluation import metrics
from vertexai.evaluation import prompt_template


EvalResult = _base.EvalResult
Expand Down
12 changes: 6 additions & 6 deletions vertexai/preview/evaluation/metrics/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,16 +16,16 @@
#
"""Evaluation Metrics Module."""

from vertexai.preview.evaluation.metrics import _base
from vertexai.preview.evaluation.metrics import _rouge
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import _base
from vertexai.evaluation.metrics import _rouge
from vertexai.evaluation.metrics import (
metric_prompt_template,
)
from vertexai.preview.evaluation.metrics import (
from vertexai.evaluation.metrics import (
metric_prompt_template_examples,
)
from vertexai.preview.evaluation.metrics import pairwise_metric
from vertexai.preview.evaluation.metrics import pointwise_metric
from vertexai.evaluation.metrics import pairwise_metric
from vertexai.evaluation.metrics import pointwise_metric


PairwiseMetric = pairwise_metric.PairwiseMetric
Expand Down

0 comments on commit 45e4251

Please sign in to comment.