Kubeflow Dashboard UI page TLS ERROR

[tppalani]

Hi All,

I'm trying to install kubeflow in my EKS cluster after deploying manifest i can able to access to Kubeflow Dashboard using port forward but after login when i'm trying click the pipeline i can see below error message.

Steps are followed to install kubeflow

export KUBEFLOW_RELEASE_VERSION=v1.7.0
export AWS_RELEASE_VERSION=v1.7.0-aws-b1.0.2
git clone https://github.com/awslabs/kubeflow-manifests.git && cd kubeflow-manifests
git checkout ${AWS_RELEASE_VERSION}
git clone --branch ${KUBEFLOW_RELEASE_VERSION} https://github.com/kubeflow/manifests.git upstream
while ! kustomize build deployments/vanilla | kubectl apply -f -; do echo "Retrying to apply resources"; sleep 10; done

Error message

upstream connect error or disconnect/reset before headers. retried and the latest reset reason: connection failure, transport failure reason: TLS error: 268435703:SSL routines:OPENSSL_internal:WRONG_VERSION_NUMBER

$ k get pod -n auth
NAME                   READY   STATUS    RESTARTS   AGE
dex-69b8795859-gd2tr   1/1     Running   0          3h23m

$ k get pod -n cert-manager
NAME                                       READY   STATUS    RESTARTS   AGE
cert-manager-8dd7db8b9-m5rb7               1/1     Running   0          3h23m
cert-manager-cainjector-56f68bbc88-l4vql   1/1     Running   0          3h23m
cert-manager-webhook-558c44c5db-cvlxz      1/1     Running   0          3h23m

$ k get pod -n istio-system
NAME                                     READY   STATUS    RESTARTS   AGE
authservice-0                            1/1     Running   0          3h23m
cluster-local-gateway-5f46cf9788-6vf92   1/1     Running   0          3h23m
istio-ingressgateway-54d7f9f654-mns6h    1/1     Running   0          3h23m
istiod-5fd8858d96-6zcj9                  1/1     Running   0          3h23m

$ k get pod -n knative-eventing
NAME                                   READY   STATUS    RESTARTS   AGE
eventing-controller-5dfdd974bd-thmpb   1/1     Running   0          3h24m
eventing-webhook-56b6c947bf-5z78r      1/1     Running   0          3h23m
eventing-webhook-56b6c947bf-6smff      1/1     Running   0          2m40s
eventing-webhook-56b6c947bf-cgdjk      1/1     Running   0          30m
eventing-webhook-56b6c947bf-lz2tc      1/1     Running   0          3h24m
eventing-webhook-56b6c947bf-qj55t      1/1     Running   0          3m25s

$ k get pod -n knative-serving
NAME                                     READY   STATUS    RESTARTS   AGE
activator-6658cbc64-hmjn6                1/1     Running   0          3h24m
autoscaler-7764d5ccdc-fdr66              1/1     Running   0          3h24m
controller-6d47c7c79d-qttmw              1/1     Running   0          3h24m
domain-mapping-bc577bcdf-czxzz           1/1     Running   0          3h24m
domainmapping-webhook-86ddd7ff98-669dw   1/1     Running   0          3h24m
net-istio-controller-dd7d966c-hkhdd      1/1     Running   0          3h24m
net-istio-webhook-5f9db855d5-p7kh4       1/1     Running   0          3h24m
webhook-6fcd8f7f95-ztfk7                 1/1     Running   0          3h24m

$ k get pod -n kubeflow
NAME                                                     READY   STATUS             RESTARTS         AGE
admission-webhook-deployment-f556f7dc-6vl9w              1/1     Running            0                3h24m
aws-kubelow-telemetry-qcbjx                              0/1     Completed          0                3h23m
cache-server-6b9bd7fc57-vhrnm                            1/1     Running            0                3h24m
centraldashboard-58c66f5cc7-ldbjg                        1/1     Running            0                3h24m
jupyter-web-app-deployment-f5f566478-vvwwd               1/1     Running            0                3h24m
katib-controller-6c964d75d7-qj866                        1/1     Running            0                3h24m
katib-db-manager-55958d4659-7cmqb                        1/1     Running            0                3h24m
katib-mysql-76b79df5b5-cp8nv                             1/1     Running            0                3h24m
katib-ui-cc85f84b5-hhkn8                                 1/1     Running            0                3h24m
kserve-controller-manager-748bc6499d-crwfn               1/2     CrashLoopBackOff   28 (2m57s ago)   3h24m
kserve-models-web-app-686df54cf5-m2lk7                   1/1     Running            0                3h24m
kubeflow-pipelines-profile-controller-5dcf75b69f-2s5wv   1/1     Running            0                3h24m
metacontroller-0                                         1/1     Running            0                3h23m
metadata-envoy-deployment-6d9dc7bb4-tjzm6                1/1     Running            0                3h24m
metadata-grpc-deployment-b45564d7d-69vfb                 1/1     Running            3 (3h23m ago)    3h24m
metadata-writer-5c6cf897d9-tncmz                         1/1     Running            0                3h24m
minio-5dc6ff5b96-7kn4x                                   1/1     Running            0                3h24m
ml-pipeline-784dbbdc5c-5ckc8                             1/1     Running            0                3h24m
ml-pipeline-persistenceagent-78c689b467-4g52h            1/1     Running            0                3h24m
ml-pipeline-scheduledworkflow-d888f97fd-mwlff            1/1     Running            0                3h24m
ml-pipeline-ui-6cf9d69c89-qgwfx                          1/1     Running            0                3h24m
ml-pipeline-viewer-crd-54d99546c9-2brcx                  1/1     Running            0                3h24m
ml-pipeline-visualizationserver-6f9977b74b-5m6tp         1/1     Running            0                3h24m
mysql-5b446b5744-jl7xs                                   1/1     Running            0                3h23m
notebook-controller-deployment-c8d48b65d-xmpgm           1/1     Running            0                3h23m
profiles-deployment-c9f84c45d-rzqc8                      2/2     Running            0                3h23m
tensorboard-controller-deployment-6d586cbf6f-mx47j       2/2     Running            0                3h23m
tensorboards-web-app-deployment-5cb7698c6d-79svr         1/1     Running            0                3h23m
training-operator-75595c5745-z7hml                       0/1     CrashLoopBackOff   59 (4m51s ago)   3h23m
volumes-web-app-deployment-7d4944f977-d2f4l              1/1     Running            0                3h23m
workflow-controller-56f497f69b-rh2g8                     1/1     Running            0                3h23m