From 6f77c2d08ba48af79b8ed7e97397c29f846e0745 Mon Sep 17 00:00:00 2001
From: Paco Xu <paco.xu@daocloud.io>
Date: Thu, 30 Mar 2023 15:38:49 +0800
Subject: [PATCH 1/5] kubeadm: init kep for no cri socket annotation

---
 .../sig-cluster-lifecycle/3929.yaml           |   3 +
 .../3929-no-cri-socket-annotation/README.md   | 772 ++++++++++++++++++
 .../3929-no-cri-socket-annotation/kep.yaml    |  16 +
 3 files changed, 791 insertions(+)
 create mode 100644 keps/prod-readiness/sig-cluster-lifecycle/3929.yaml
 create mode 100644 keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
 create mode 100644 keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml

diff --git a/keps/prod-readiness/sig-cluster-lifecycle/3929.yaml b/keps/prod-readiness/sig-cluster-lifecycle/3929.yaml
new file mode 100644
index 00000000000..f9a687a8007
--- /dev/null
+++ b/keps/prod-readiness/sig-cluster-lifecycle/3929.yaml
@@ -0,0 +1,3 @@
+kep-number: 3929
+alpha:
+  approver: "@"
diff --git a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
new file mode 100644
index 00000000000..2c15263b40a
--- /dev/null
+++ b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
@@ -0,0 +1,772 @@
+<!--
+**Note:** When your KEP is complete, all of these comment blocks should be removed.
+
+To get started with this template:
+
+- [ ] **Pick a hosting SIG.**
+  Make sure that the problem space is something the SIG is interested in taking
+  up. KEPs should not be checked in without a sponsoring SIG.
+- [ ] **Create an issue in kubernetes/enhancements**
+  When filing an enhancement tracking issue, please make sure to complete all
+  fields in that template. One of the fields asks for a link to the KEP. You
+  can leave that blank until this KEP is filed, and then go back to the
+  enhancement and add the link.
+- [ ] **Make a copy of this template directory.**
+  Copy this template into the owning SIG's directory and name it
+  `NNNN-short-descriptive-title`, where `NNNN` is the issue number (with no
+  leading-zero padding) assigned to your enhancement above.
+- [ ] **Fill out as much of the kep.yaml file as you can.**
+  At minimum, you should fill in the "Title", "Authors", "Owning-sig",
+  "Status", and date-related fields.
+- [ ] **Fill out this file as best you can.**
+  At minimum, you should fill in the "Summary" and "Motivation" sections.
+  These should be easy if you've preflighted the idea of the KEP with the
+  appropriate SIG(s).
+- [ ] **Create a PR for this KEP.**
+  Assign it to people in the SIG who are sponsoring this process.
+- [ ] **Merge early and iterate.**
+  Avoid getting hung up on specific details and instead aim to get the goals of
+  the KEP clarified and merged quickly. The best way to do this is to just
+  start with the high-level sections and fill out details incrementally in
+  subsequent PRs.
+
+Just because a KEP is merged does not mean it is complete or approved. Any KEP
+marked as `provisional` is a working document and subject to change. You can
+denote sections that are under active debate as follows:
+
+```
+<<[UNRESOLVED optional short context or usernames ]>>
+Stuff that is being argued.
+<<[/UNRESOLVED]>>
+```
+
+When editing KEPS, aim for tightly-scoped, single-topic PRs to keep discussions
+focused. If you disagree with what is already in a document, open a new PR
+with suggested changes.
+
+One KEP corresponds to one "feature" or "enhancement" for its whole lifecycle.
+You do not need a new KEP to move from beta to GA, for example. If
+new details emerge that belong in the KEP, edit the KEP. Once a feature has become
+"implemented", major changes should get new KEPs.
+
+The canonical place for the latest set of instructions (and the likely source
+of this file) is [here](/keps/NNNN-kep-template/README.md).
+
+**Note:** Any PRs to move a KEP to `implementable`, or significant changes once
+it is marked `implementable`, must be approved by each of the KEP approvers.
+If none of those approvers are still appropriate, then changes to that list
+should be approved by the remaining approvers and/or the owning SIG (or
+SIG Architecture for cross-cutting KEPs).
+-->
+# 3929: Remove CRI Socket Annotation from Node Object
+
+<!--
+This is the title of your KEP. Keep it short, simple, and descriptive. A good
+title can help communicate what the KEP is and should be considered as part of
+any review.
+-->
+
+<!--
+A table of contents is helpful for quickly jumping to sections of a KEP and for
+highlighting any additional information provided beyond the standard KEP
+template.
+
+Ensure the TOC is wrapped with
+  <code>&lt;!-- toc --&rt;&lt;!-- /toc --&rt;</code>
+tags, and then generate with `hack/update-toc.sh`.
+-->
+
+<!-- toc -->
+- [Release Signoff Checklist](#release-signoff-checklist)
+- [Summary](#summary)
+- [Motivation](#motivation)
+  - [Goals](#goals)
+  - [Non-Goals](#non-goals)
+- [Proposal](#proposal)
+  - [User Stories (Optional)](#user-stories-optional)
+    - [Story 1](#story-1)
+    - [Story 2](#story-2)
+  - [Notes/Constraints/Caveats (Optional)](#notesconstraintscaveats-optional)
+  - [Risks and Mitigations](#risks-and-mitigations)
+- [Design Details](#design-details)
+  - [Test Plan](#test-plan)
+    - [Prerequisite testing updates](#prerequisite-testing-updates)
+    - [Unit tests](#unit-tests)
+    - [Integration tests](#integration-tests)
+    - [e2e tests](#e2e-tests)
+  - [Graduation Criteria](#graduation-criteria)
+  - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)
+  - [Version Skew Strategy](#version-skew-strategy)
+- [Production Readiness Review Questionnaire](#production-readiness-review-questionnaire)
+  - [Feature Enablement and Rollback](#feature-enablement-and-rollback)
+  - [Rollout, Upgrade and Rollback Planning](#rollout-upgrade-and-rollback-planning)
+  - [Monitoring Requirements](#monitoring-requirements)
+  - [Dependencies](#dependencies)
+  - [Scalability](#scalability)
+  - [Troubleshooting](#troubleshooting)
+- [Implementation History](#implementation-history)
+- [Drawbacks](#drawbacks)
+- [Alternatives](#alternatives)
+- [Infrastructure Needed (Optional)](#infrastructure-needed-optional)
+<!-- /toc -->
+
+## Release Signoff Checklist
+
+<!--
+**ACTION REQUIRED:** In order to merge code into a release, there must be an
+issue in [kubernetes/enhancements] referencing this KEP and targeting a release
+milestone **before the [Enhancement Freeze](https://git.k8s.io/sig-release/releases)
+of the targeted release**.
+
+For enhancements that make changes to code or processes/procedures in core
+Kubernetes—i.e., [kubernetes/kubernetes], we require the following Release
+Signoff checklist to be completed.
+
+Check these off as they are completed for the Release Team to track. These
+checklist items _must_ be updated for the enhancement to be released.
+-->
+
+Items marked with (R) are required *prior to targeting to a milestone / release*.
+
+- [ ] (R) Enhancement issue in release milestone, which links to KEP dir in [kubernetes/enhancements] (not the initial KEP PR)
+- [ ] (R) KEP approvers have approved the KEP status as `implementable`
+- [ ] (R) Design details are appropriately documented
+- [ ] (R) Test plan is in place, giving consideration to SIG Architecture and SIG Testing input (including test refactors)
+  - [ ] e2e Tests for all Beta API Operations (endpoints)
+  - [ ] (R) Ensure GA e2e tests meet requirements for [Conformance Tests](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md)
+  - [ ] (R) Minimum Two Week Window for GA e2e tests to prove flake free
+- [ ] (R) Graduation criteria is in place
+  - [ ] (R) [all GA Endpoints](https://github.com/kubernetes/community/pull/1806) must be hit by [Conformance Tests](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md)
+- [ ] (R) Production readiness review completed
+- [ ] (R) Production readiness review approved
+- [ ] "Implementation History" section is up-to-date for milestone
+- [ ] User-facing documentation has been created in [kubernetes/website], for publication to [kubernetes.io]
+- [ ] Supporting documentation—e.g., additional design documents, links to mailing list discussions/SIG meetings, relevant PRs/issues, release notes
+
+<!--
+**Note:** This checklist is iterative and should be reviewed and updated every time this enhancement is being considered for a milestone.
+-->
+
+[kubernetes.io]: https://kubernetes.io/
+[kubernetes/enhancements]: https://git.k8s.io/enhancements
+[kubernetes/website]: https://git.k8s.io/website
+
+## Summary
+
+The proposal is to remove the Container Runtime Interface (CRI) socket annotation
+from the Node object in Kubernetes, which is currently added during the
+"init upload-config" phase in Kubeadm. This annotation is used to specify the CRI
+socket endpoint used by the kubelet on each node for communication with the container
+runtime. Instead of relying on this annotation, the proposal suggests using a global
+kubelet configuration with a CRI socket specified, as well as providing the ability to
+override this configuration during kubeadm join using the --config flag. This would
+eliminate the need for kubeadm to store CRI socket configuration on each node, and
+instead rely on the Kubernetes configuration files for specifying this information.
+
+## Motivation
+
+Currently, kubeadm adds a CRI socket annotation to the Node object during the
+"init upload-config" phase, which specifies the endpoint for the CRI that is being
+used by the kubelet on each node. This annotation is persistent on the Node object,
+even if the kubelet is updated or the CRI is changed.
+
+After migration of container runtime endpoint flag to kubelet config, we can set
+cri socket in kubelet configuration.
+
+### Goals
+
+- Remove the use of CRI socket annotation on Node object
+- For node customized kubelet configuration, it can be saved locally on disk with file path
+  `/var/lib/kubelet/kubeadm-config.yaml` and we will prioritize respecting the local
+  setting over the global one.
+
+### Non-Goals
+
+- Update the CRI socket annotation on Node object to be the latest
+
+## Proposal
+
+1. init: upload a global kubelet configuration with cri socket
+2. join: can override it using --config
+3. upgrade: re-download global one, but should use local kubelet configuration firstly
+
+### User Stories (Optional)
+
+#### Story 1
+
+#### Story 2
+
+### Notes/Constraints/Caveats (Optional)
+
+### Risks and Mitigations
+
+## Design Details
+
+### init: upload a global kubelet configuration with cri socket
+
+- `kubeadm init` will not add the annotation to node.
+- `kubeadm init` will check the customized `--config` at first and if no cri socket is set, it will
+  auto-detect it and save it global configuration and local as well.
+
+### join: can override it using --config
+
+- `kubeadm join` will not add the annotation to node.
+- `kubeadm join` will download the kubelet configuration from apiserver and the customized `--config`
+  at first and auto-detect will work only if not set. Auto-detect may log a warning message if it may
+  be misconfigured and log a general debug log if there is multi CRI-sockets.
+
+### upgrade: re-download global one, but should use local kubelet configuration firstly
+
+- `kubeadm upgrade` will download the kubelet configuration from apiserver and respect local one.
+
+### old version handling
+
+For old version cluster upgradation with the annotation, we will not touch the annotation at first.
+
+1. in v1.28, `kubeadm upgrade` will respect the annotation and save it to `/var/lib/kubelet/kubeadm-config.yaml`.
+2. in v1.29, `kubeadm upgrade` will ignore the annotation.
+
+### Test Plan
+
+<!--
+**Note:** *Not required until targeted at a release.*
+The goal is to ensure that we don't accept enhancements with inadequate testing.
+
+All code is expected to have adequate tests (eventually with coverage
+expectations). Please adhere to the [Kubernetes testing guidelines][testing-guidelines]
+when drafting this test plan.
+
+[testing-guidelines]: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
+-->
+
+[ ] I/we understand the owners of the involved components may require updates to
+existing tests to make this code solid enough prior to committing the changes necessary
+to implement this enhancement.
+
+##### Prerequisite testing updates
+
+<!--
+Based on reviewers feedback describe what additional tests need to be added prior
+implementing this enhancement to ensure the enhancements have also solid foundations.
+-->
+
+##### Unit tests
+
+<!--
+In principle every added code should have complete unit test coverage, so providing
+the exact set of tests will not bring additional value.
+However, if complete unit test coverage is not possible, explain the reason of it
+together with explanation why this is acceptable.
+-->
+
+<!--
+Additionally, for Alpha try to enumerate the core package you will be touching
+to implement this enhancement and provide the current unit coverage for those
+in the form of:
+- <package>: <date> - <current test coverage>
+The data can be easily read from:
+https://testgrid.k8s.io/sig-testing-canaries#ci-kubernetes-coverage-unit
+
+This can inform certain test coverage improvements that we want to do before
+extending the production code to implement this enhancement.
+-->
+
+- `<package>`: `<date>` - `<test coverage>`
+
+##### Integration tests
+
+<!--
+This question should be filled when targeting a release.
+For Alpha, describe what tests will be added to ensure proper quality of the enhancement.
+
+For Beta and GA, add links to added tests together with links to k8s-triage for those tests:
+https://storage.googleapis.com/k8s-triage/index.html
+-->
+
+- <test>: <link to test coverage>
+
+##### e2e tests
+
+<!--
+This question should be filled when targeting a release.
+For Alpha, describe what tests will be added to ensure proper quality of the enhancement.
+
+For Beta and GA, add links to added tests together with links to k8s-triage for those tests:
+https://storage.googleapis.com/k8s-triage/index.html
+
+We expect no non-infra related flakes in the last month as a GA graduation criteria.
+-->
+
+- <test>: <link to test coverage>
+
+### Graduation Criteria
+
+<!--
+**Note:** *Not required until targeted at a release.*
+
+Define graduation milestones.
+
+These may be defined in terms of API maturity, [feature gate] graduations, or as
+something else. The KEP should keep this high-level with a focus on what
+signals will be looked at to determine graduation.
+
+Consider the following in developing the graduation criteria for this enhancement:
+- [Maturity levels (`alpha`, `beta`, `stable`)][maturity-levels]
+- [Feature gate][feature gate] lifecycle
+- [Deprecation policy][deprecation-policy]
+
+Clearly define what graduation means by either linking to the [API doc
+definition](https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-versioning)
+or by redefining what graduation means.
+
+In general we try to use the same stages (alpha, beta, GA), regardless of how the
+functionality is accessed.
+
+[feature gate]: https://git.k8s.io/community/contributors/devel/sig-architecture/feature-gates.md
+[maturity-levels]: https://git.k8s.io/community/contributors/devel/sig-architecture/api_changes.md#alpha-beta-and-stable-versions
+[deprecation-policy]: https://kubernetes.io/docs/reference/using-api/deprecation-policy/
+
+Below are some examples to consider, in addition to the aforementioned [maturity levels][maturity-levels].
+
+#### Alpha
+
+- Feature implemented behind a feature flag
+- Initial e2e tests completed and enabled
+
+#### Beta
+
+- Gather feedback from developers and surveys
+- Complete features A, B, C
+- Additional tests are in Testgrid and linked in KEP
+
+#### GA
+
+- N examples of real-world usage
+- N installs
+- More rigorous forms of testing—e.g., downgrade tests and scalability tests
+- Allowing time for feedback
+
+**Note:** Generally we also wait at least two releases between beta and
+GA/stable, because there's no opportunity for user feedback, or even bug reports,
+in back-to-back releases.
+
+**For non-optional features moving to GA, the graduation criteria must include
+[conformance tests].**
+
+[conformance tests]: https://git.k8s.io/community/contributors/devel/sig-architecture/conformance-tests.md
+
+#### Deprecation
+
+- Announce deprecation and support policy of the existing flag
+- Two versions passed since introducing the functionality that deprecates the flag (to address version skew)
+- Address feedback on usage/changed behavior, provided on GitHub issues
+- Deprecate the flag
+-->
+
+### Upgrade / Downgrade Strategy
+
+<!--
+If applicable, how will the component be upgraded and downgraded? Make sure
+this is in the test plan.
+
+Consider the following in developing an upgrade/downgrade strategy for this
+enhancement:
+- What changes (in invocations, configurations, API use, etc.) is an existing
+  cluster required to make on upgrade, in order to maintain previous behavior?
+- What changes (in invocations, configurations, API use, etc.) is an existing
+  cluster required to make on upgrade, in order to make use of the enhancement?
+-->
+
+### Version Skew Strategy
+
+<!--
+If applicable, how will the component handle version skew with other
+components? What are the guarantees? Make sure this is in the test plan.
+
+Consider the following in developing a version skew strategy for this
+enhancement:
+- Does this enhancement involve coordinating behavior in the control plane and
+  in the kubelet? How does an n-2 kubelet without this feature available behave
+  when this feature is used?
+- Will any other components on the node change? For example, changes to CSI,
+  CRI or CNI may require updating that component before the kubelet.
+-->
+
+## Production Readiness Review Questionnaire
+
+<!--
+
+Production readiness reviews are intended to ensure that features merging into
+Kubernetes are observable, scalable and supportable; can be safely operated in
+production environments, and can be disabled or rolled back in the event they
+cause increased failures in production. See more in the PRR KEP at
+https://git.k8s.io/enhancements/keps/sig-architecture/1194-prod-readiness.
+
+The production readiness review questionnaire must be completed and approved
+for the KEP to move to `implementable` status and be included in the release.
+
+In some cases, the questions below should also have answers in `kep.yaml`. This
+is to enable automation to verify the presence of the review, and to reduce review
+burden and latency.
+
+The KEP must have a approver from the
+[`prod-readiness-approvers`](http://git.k8s.io/enhancements/OWNERS_ALIASES)
+team. Please reach out on the
+[#prod-readiness](https://kubernetes.slack.com/archives/CPNHUMN74) channel if
+you need any help or guidance.
+-->
+
+### Feature Enablement and Rollback
+
+<!--
+This section must be completed when targeting alpha to a release.
+-->
+
+###### How can this feature be enabled / disabled in a live cluster?
+
+<!--
+Pick one of these and delete the rest.
+
+Documentation is available on [feature gate lifecycle] and expectations, as
+well as the [existing list] of feature gates.
+
+[feature gate lifecycle]: https://git.k8s.io/community/contributors/devel/sig-architecture/feature-gates.md
+[existing list]: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
+-->
+
+- [ ] Feature gate (also fill in values in `kep.yaml`)
+  - Feature gate name:
+  - Components depending on the feature gate:
+- [ ] Other
+  - Describe the mechanism:
+  - Will enabling / disabling the feature require downtime of the control
+    plane?
+  - Will enabling / disabling the feature require downtime or reprovisioning
+    of a node? (Do not assume `Dynamic Kubelet Config` feature is enabled).
+
+###### Does enabling the feature change any default behavior?
+
+<!--
+Any change of default behavior may be surprising to users or break existing
+automations, so be extremely careful here.
+-->
+
+###### Can the feature be disabled once it has been enabled (i.e. can we roll back the enablement)?
+
+<!--
+Describe the consequences on existing workloads (e.g., if this is a runtime
+feature, can it break the existing applications?).
+
+Feature gates are typically disabled by setting the flag to `false` and
+restarting the component. No other changes should be necessary to disable the
+feature.
+
+NOTE: Also set `disable-supported` to `true` or `false` in `kep.yaml`.
+-->
+
+###### What happens if we reenable the feature if it was previously rolled back?
+
+###### Are there any tests for feature enablement/disablement?
+
+<!--
+The e2e framework does not currently support enabling or disabling feature
+gates. However, unit tests in each component dealing with managing data, created
+with and without the feature, are necessary. At the very least, think about
+conversion tests if API types are being modified.
+
+Additionally, for features that are introducing a new API field, unit tests that
+are exercising the `switch` of feature gate itself (what happens if I disable a
+feature gate after having objects written with the new field) are also critical.
+You can take a look at one potential example of such test in:
+https://github.com/kubernetes/kubernetes/pull/97058/files#diff-7826f7adbc1996a05ab52e3f5f02429e94b68ce6bce0dc534d1be636154fded3R246-R282
+-->
+
+### Rollout, Upgrade and Rollback Planning
+
+<!--
+This section must be completed when targeting beta to a release.
+-->
+
+###### How can a rollout or rollback fail? Can it impact already running workloads?
+
+<!--
+Try to be as paranoid as possible - e.g., what if some components will restart
+mid-rollout?
+
+Be sure to consider highly-available clusters, where, for example,
+feature flags will be enabled on some API servers and not others during the
+rollout. Similarly, consider large clusters and how enablement/disablement
+will rollout across nodes.
+-->
+
+###### What specific metrics should inform a rollback?
+
+<!--
+What signals should users be paying attention to when the feature is young
+that might indicate a serious problem?
+-->
+
+###### Were upgrade and rollback tested? Was the upgrade->downgrade->upgrade path tested?
+
+<!--
+Describe manual testing that was done and the outcomes.
+Longer term, we may want to require automated upgrade/rollback tests, but we
+are missing a bunch of machinery and tooling and can't do that now.
+-->
+
+###### Is the rollout accompanied by any deprecations and/or removals of features, APIs, fields of API types, flags, etc.?
+
+<!--
+Even if applying deprecation policies, they may still surprise some users.
+-->
+
+### Monitoring Requirements
+
+<!--
+This section must be completed when targeting beta to a release.
+
+For GA, this section is required: approvers should be able to confirm the
+previous answers based on experience in the field.
+-->
+
+###### How can an operator determine if the feature is in use by workloads?
+
+<!--
+Ideally, this should be a metric. Operations against the Kubernetes API (e.g.,
+checking if there are objects with field X set) may be a last resort. Avoid
+logs or events for this purpose.
+-->
+
+###### How can someone using this feature know that it is working for their instance?
+
+<!--
+For instance, if this is a pod-related feature, it should be possible to determine if the feature is functioning properly
+for each individual pod.
+Pick one more of these and delete the rest.
+Please describe all items visible to end users below with sufficient detail so that they can verify correct enablement
+and operation of this feature.
+Recall that end users cannot usually observe component logs or access metrics.
+-->
+
+- [ ] Events
+  - Event Reason:
+- [ ] API .status
+  - Condition name:
+  - Other field:
+- [ ] Other (treat as last resort)
+  - Details:
+
+###### What are the reasonable SLOs (Service Level Objectives) for the enhancement?
+
+<!--
+This is your opportunity to define what "normal" quality of service looks like
+for a feature.
+
+It's impossible to provide comprehensive guidance, but at the very
+high level (needs more precise definitions) those may be things like:
+  - per-day percentage of API calls finishing with 5XX errors <= 1%
+  - 99% percentile over day of absolute value from (job creation time minus expected
+    job creation time) for cron job <= 10%
+  - 99.9% of /health requests per day finish with 200 code
+
+These goals will help you determine what you need to measure (SLIs) in the next
+question.
+-->
+
+###### What are the SLIs (Service Level Indicators) an operator can use to determine the health of the service?
+
+<!--
+Pick one more of these and delete the rest.
+-->
+
+- [ ] Metrics
+  - Metric name:
+  - [Optional] Aggregation method:
+  - Components exposing the metric:
+- [ ] Other (treat as last resort)
+  - Details:
+
+###### Are there any missing metrics that would be useful to have to improve observability of this feature?
+
+<!--
+Describe the metrics themselves and the reasons why they weren't added (e.g., cost,
+implementation difficulties, etc.).
+-->
+
+### Dependencies
+
+<!--
+This section must be completed when targeting beta to a release.
+-->
+
+###### Does this feature depend on any specific services running in the cluster?
+
+<!--
+Think about both cluster-level services (e.g. metrics-server) as well
+as node-level agents (e.g. specific version of CRI). Focus on external or
+optional services that are needed. For example, if this feature depends on
+a cloud provider API, or upon an external software-defined storage or network
+control plane.
+
+For each of these, fill in the following—thinking about running existing user workloads
+and creating new ones, as well as about cluster-level services (e.g. DNS):
+  - [Dependency name]
+    - Usage description:
+      - Impact of its outage on the feature:
+      - Impact of its degraded performance or high-error rates on the feature:
+-->
+
+### Scalability
+
+<!--
+For alpha, this section is encouraged: reviewers should consider these questions
+and attempt to answer them.
+
+For beta, this section is required: reviewers must answer these questions.
+
+For GA, this section is required: approvers should be able to confirm the
+previous answers based on experience in the field.
+-->
+
+###### Will enabling / using this feature result in any new API calls?
+
+<!--
+Describe them, providing:
+  - API call type (e.g. PATCH pods)
+  - estimated throughput
+  - originating component(s) (e.g. Kubelet, Feature-X-controller)
+Focusing mostly on:
+  - components listing and/or watching resources they didn't before
+  - API calls that may be triggered by changes of some Kubernetes resources
+    (e.g. update of object X triggers new updates of object Y)
+  - periodic API calls to reconcile state (e.g. periodic fetching state,
+    heartbeats, leader election, etc.)
+-->
+
+###### Will enabling / using this feature result in introducing new API types?
+
+<!--
+Describe them, providing:
+  - API type
+  - Supported number of objects per cluster
+  - Supported number of objects per namespace (for namespace-scoped objects)
+-->
+
+###### Will enabling / using this feature result in any new calls to the cloud provider?
+
+<!--
+Describe them, providing:
+  - Which API(s):
+  - Estimated increase:
+-->
+
+###### Will enabling / using this feature result in increasing size or count of the existing API objects?
+
+<!--
+Describe them, providing:
+  - API type(s):
+  - Estimated increase in size: (e.g., new annotation of size 32B)
+  - Estimated amount of new objects: (e.g., new Object X for every existing Pod)
+-->
+
+###### Will enabling / using this feature result in increasing time taken by any operations covered by existing SLIs/SLOs?
+
+<!--
+Look at the [existing SLIs/SLOs].
+
+Think about adding additional work or introducing new steps in between
+(e.g. need to do X to start a container), etc. Please describe the details.
+
+[existing SLIs/SLOs]: https://git.k8s.io/community/sig-scalability/slos/slos.md#kubernetes-slisslos
+-->
+
+###### Will enabling / using this feature result in non-negligible increase of resource usage (CPU, RAM, disk, IO, ...) in any components?
+
+<!--
+Things to keep in mind include: additional in-memory state, additional
+non-trivial computations, excessive access to disks (including increased log
+volume), significant amount of data sent and/or received over network, etc.
+This through this both in small and large cases, again with respect to the
+[supported limits].
+
+[supported limits]: https://git.k8s.io/community//sig-scalability/configs-and-limits/thresholds.md
+-->
+
+###### Can enabling / using this feature result in resource exhaustion of some node resources (PIDs, sockets, inodes, etc.)?
+
+<!--
+Focus not just on happy cases, but primarily on more pathological cases
+(e.g. probes taking a minute instead of milliseconds, failed pods consuming resources, etc.).
+If any of the resources can be exhausted, how this is mitigated with the existing limits
+(e.g. pods per node) or new limits added by this KEP?
+
+Are there any tests that were run/should be run to understand performance characteristics better
+and validate the declared limits?
+-->
+
+### Troubleshooting
+
+<!--
+This section must be completed when targeting beta to a release.
+
+For GA, this section is required: approvers should be able to confirm the
+previous answers based on experience in the field.
+
+The Troubleshooting section currently serves the `Playbook` role. We may consider
+splitting it into a dedicated `Playbook` document (potentially with some monitoring
+details). For now, we leave it here.
+-->
+
+###### How does this feature react if the API server and/or etcd is unavailable?
+
+###### What are other known failure modes?
+
+<!--
+For each of them, fill in the following information by copying the below template:
+  - [Failure mode brief description]
+    - Detection: How can it be detected via metrics? Stated another way:
+      how can an operator troubleshoot without logging into a master or worker node?
+    - Mitigations: What can be done to stop the bleeding, especially for already
+      running user workloads?
+    - Diagnostics: What are the useful log messages and their required logging
+      levels that could help debug the issue?
+      Not required until feature graduated to beta.
+    - Testing: Are there any tests for failure mode? If not, describe why.
+-->
+
+###### What steps should be taken if SLOs are not being met to determine the problem?
+
+## Implementation History
+
+<!--
+Major milestones in the lifecycle of a KEP should be tracked in this section.
+Major milestones might include:
+- the `Summary` and `Motivation` sections being merged, signaling SIG acceptance
+- the `Proposal` section being merged, signaling agreement on a proposed design
+- the date implementation started
+- the first Kubernetes release where an initial version of the KEP was available
+- the version of Kubernetes where the KEP graduated to general availability
+- when the KEP was retired or superseded
+-->
+
+## Drawbacks
+
+<!--
+Why should this KEP _not_ be implemented?
+-->
+
+## Alternatives
+
+<!--
+What other approaches did you consider, and why did you rule them out? These do
+not need to be as detailed as the proposal, but should include enough
+information to express the idea and why it was not acceptable.
+-->
+
+## Infrastructure Needed (Optional)
+
+<!--
+Use this section if you need things from the project/SIG. Examples include a
+new subproject, repos requested, or GitHub details. Listing these here allows a
+SIG to get the process for these resources started right away.
+-->
diff --git a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml
new file mode 100644
index 00000000000..403320c0338
--- /dev/null
+++ b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml
@@ -0,0 +1,16 @@
+title: "Remove CRI Socket Annotation from Node Object"
+kep-number: 3929
+authors:
+  - "@pacoxu"
+owning-sig: sig-cluster-lifecycle
+participating-sigs:
+  - sig-cluster-lifecycle
+status: provisional
+creation-date: 2023-03-30
+last-updated: 2022-03-30
+reviewers:
+  - "@neolit123"
+approvers:
+  - "@neolit123"
+latest-milestone: "0.0"
+stage: "alpha"

From c77f42ddcc13d3fc8e01c15670dccb9260a80089 Mon Sep 17 00:00:00 2001
From: Paco Xu <paco.xu@daocloud.io>
Date: Thu, 30 Mar 2023 17:51:39 +0800
Subject: [PATCH 2/5] add some details to make the kep more substantial

---
 .../3929-no-cri-socket-annotation/README.md   | 80 ++++++-------------
 1 file changed, 25 insertions(+), 55 deletions(-)

diff --git a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
index 2c15263b40a..c4bfbddb132 100644
--- a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
+++ b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
@@ -128,9 +128,9 @@ checklist items _must_ be updated for the enhancement to be released.
 
 Items marked with (R) are required *prior to targeting to a milestone / release*.
 
-- [ ] (R) Enhancement issue in release milestone, which links to KEP dir in [kubernetes/enhancements] (not the initial KEP PR)
+- [x] (R) Enhancement issue in release milestone, which links to KEP dir in [kubernetes/enhancements] (not the initial KEP PR)
 - [ ] (R) KEP approvers have approved the KEP status as `implementable`
-- [ ] (R) Design details are appropriately documented
+- [x] (R) Design details are appropriately documented
 - [ ] (R) Test plan is in place, giving consideration to SIG Architecture and SIG Testing input (including test refactors)
   - [ ] e2e Tests for all Beta API Operations (endpoints)
   - [ ] (R) Ensure GA e2e tests meet requirements for [Conformance Tests](https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/conformance-tests.md)
@@ -202,6 +202,16 @@ cri socket in kubelet configuration.
 
 ## Design Details
 
+We should introduce a `/var/lib/kubelet/kubeadm-config.yaml` to maintain node specific configuration.
+It is similar to `/var/lib/kubelet/kubeadm-flags.env`.
+
+```text
+KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=k8s.m.daocloud.io/pause:3.9"
+```
+
+[To be discussed] Another proposal is using a strategy like `--patch`. A file like `/var/lib/kubelet/kubeadm-config.patch`
+or a `kubelet.yaml`/`config.ayml` file under `/var/lib/kubelet/patch/`. (This should be removed if we make a decision).
+
 ### init: upload a global kubelet configuration with cri socket
 
 - `kubeadm init` will not add the annotation to node.
@@ -239,10 +249,15 @@ when drafting this test plan.
 [testing-guidelines]: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
 -->
 
-[ ] I/we understand the owners of the involved components may require updates to
+[x] I/we understand the owners of the involved components may require updates to
 existing tests to make this code solid enough prior to committing the changes necessary
 to implement this enhancement.
 
+Install/Join/Upgrade test in <https://testgrid.k8s.io/sig-cluster-lifecycle-kubeadm>
+
+- upgrade v1.(n-1) to v1.n.
+- upgrade v1.n to v1.n.
+
 ##### Prerequisite testing updates
 
 <!--
@@ -301,49 +316,20 @@ We expect no non-infra related flakes in the last month as a GA graduation crite
 
 ### Graduation Criteria
 
-<!--
-**Note:** *Not required until targeted at a release.*
-
-Define graduation milestones.
-
-These may be defined in terms of API maturity, [feature gate] graduations, or as
-something else. The KEP should keep this high-level with a focus on what
-signals will be looked at to determine graduation.
-
-Consider the following in developing the graduation criteria for this enhancement:
-- [Maturity levels (`alpha`, `beta`, `stable`)][maturity-levels]
-- [Feature gate][feature gate] lifecycle
-- [Deprecation policy][deprecation-policy]
-
-Clearly define what graduation means by either linking to the [API doc
-definition](https://kubernetes.io/docs/concepts/overview/kubernetes-api/#api-versioning)
-or by redefining what graduation means.
-
-In general we try to use the same stages (alpha, beta, GA), regardless of how the
-functionality is accessed.
-
-[feature gate]: https://git.k8s.io/community/contributors/devel/sig-architecture/feature-gates.md
-[maturity-levels]: https://git.k8s.io/community/contributors/devel/sig-architecture/api_changes.md#alpha-beta-and-stable-versions
-[deprecation-policy]: https://kubernetes.io/docs/reference/using-api/deprecation-policy/
-
-Below are some examples to consider, in addition to the aforementioned [maturity levels][maturity-levels].
-
 #### Alpha
 
-- Feature implemented behind a feature flag
+- The upgrade will still respect the CRI annotation
 - Initial e2e tests completed and enabled
 
 #### Beta
 
+- Use the local kubelet configuration or global configuration, ignore the CRI annotation
 - Gather feedback from developers and surveys
-- Complete features A, B, C
 - Additional tests are in Testgrid and linked in KEP
 
 #### GA
 
-- N examples of real-world usage
-- N installs
-- More rigorous forms of testing—e.g., downgrade tests and scalability tests
+- Remove the CRI annotation during upgrade(this may be not urgent or have to)
 - Allowing time for feedback
 
 **Note:** Generally we also wait at least two releases between beta and
@@ -357,25 +343,9 @@ in back-to-back releases.
 
 #### Deprecation
 
-- Announce deprecation and support policy of the existing flag
-- Two versions passed since introducing the functionality that deprecates the flag (to address version skew)
-- Address feedback on usage/changed behavior, provided on GitHub issues
-- Deprecate the flag
--->
-
 ### Upgrade / Downgrade Strategy
 
-<!--
-If applicable, how will the component be upgraded and downgraded? Make sure
-this is in the test plan.
-
-Consider the following in developing an upgrade/downgrade strategy for this
-enhancement:
-- What changes (in invocations, configurations, API use, etc.) is an existing
-  cluster required to make on upgrade, in order to maintain previous behavior?
-- What changes (in invocations, configurations, API use, etc.) is an existing
-  cluster required to make on upgrade, in order to make use of the enhancement?
--->
+See above.
 
 ### Version Skew Strategy
 
@@ -435,14 +405,14 @@ well as the [existing list] of feature gates.
 -->
 
 - [ ] Feature gate (also fill in values in `kep.yaml`)
-  - Feature gate name:
+  - Feature gate name: No
   - Components depending on the feature gate:
 - [ ] Other
   - Describe the mechanism:
   - Will enabling / disabling the feature require downtime of the control
-    plane?
+    plane? No.
   - Will enabling / disabling the feature require downtime or reprovisioning
-    of a node? (Do not assume `Dynamic Kubelet Config` feature is enabled).
+    of a node? No.
 
 ###### Does enabling the feature change any default behavior?
 

From 5c6d843f349225d8ca181a1f91ff78d3525fb9ee Mon Sep 17 00:00:00 2001
From: Paco Xu <paco.xu@daocloud.io>
Date: Fri, 31 Mar 2023 10:46:20 +0800
Subject: [PATCH 3/5] add two proposal: a node-specific overwrite list or a
 node specific yaml

---
 .../3929-no-cri-socket-annotation/README.md   | 42 ++++++++++++-------
 1 file changed, 28 insertions(+), 14 deletions(-)

diff --git a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
index c4bfbddb132..c813da06c81 100644
--- a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
+++ b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
@@ -176,9 +176,9 @@ cri socket in kubelet configuration.
 ### Goals
 
 - Remove the use of CRI socket annotation on Node object
-- For node customized kubelet configuration, it can be saved locally on disk with file path
-  `/var/lib/kubelet/kubeadm-config.yaml` and we will prioritize respecting the local
-  setting over the global one.
+- We will prioritize respecting the local setting over the global one.
+- [Not decided yet] For further node customized kubelet configuration, it can be saved locally
+  on disk with file path `/var/lib/kubelet/kubeadm-config.yaml`. (If not used, I will move it to Non-Goals)
 
 ### Non-Goals
 
@@ -202,16 +202,6 @@ cri socket in kubelet configuration.
 
 ## Design Details
 
-We should introduce a `/var/lib/kubelet/kubeadm-config.yaml` to maintain node specific configuration.
-It is similar to `/var/lib/kubelet/kubeadm-flags.env`.
-
-```text
-KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=k8s.m.daocloud.io/pause:3.9"
-```
-
-[To be discussed] Another proposal is using a strategy like `--patch`. A file like `/var/lib/kubelet/kubeadm-config.patch`
-or a `kubelet.yaml`/`config.ayml` file under `/var/lib/kubelet/patch/`. (This should be removed if we make a decision).
-
 ### init: upload a global kubelet configuration with cri socket
 
 - `kubeadm init` will not add the annotation to node.
@@ -229,11 +219,35 @@ or a `kubelet.yaml`/`config.ayml` file under `/var/lib/kubelet/patch/`. (This sh
 
 - `kubeadm upgrade` will download the kubelet configuration from apiserver and respect local one.
 
+### Proposal 1: respect a list of configuration in local kubelet configuration, and in v1.27, CRI socket is the only one
+
+During `kubeadm ugprade`, kubeadm will read the local kubelet configuration in `/var/lib/kubelet/config.yaml`.
+kubeadm also download the kubelet configuration from configmap and replace the `containerRuntimeEndpoint` and
+`imageServiceEndpoint`(This maybe empty and I prefer to respect it as well) with the local configuration.
+
+A node-specific kubelet configuration list should be maintained in kubeadm code.
+
+- containerRuntimeEndpoint
+- imageServiceEndpoint
+
+### Proposal 2: introduce a `/var/lib/kubelet/kubeadm-config.yaml` to maintain node specific configuration
+
+We should introduce a `/var/lib/kubelet/kubeadm-config.yaml` to maintain node specific configuration.
+It is similar to `/var/lib/kubelet/kubeadm-flags.env`.
+
+```text
+KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=k8s.m.daocloud.io/pause:3.9"
+```
+
+[To be discussed] Another proposal is using a strategy like `--patch`. A file like `/var/lib/kubelet/kubeadm-config.patch`
+or a `kubelet.yaml`/`config.ayml` file under `/var/lib/kubelet/patch/`. (This should be removed if we make a decision).
+
 ### old version handling
 
 For old version cluster upgradation with the annotation, we will not touch the annotation at first.
 
-1. in v1.28, `kubeadm upgrade` will respect the annotation and save it to `/var/lib/kubelet/kubeadm-config.yaml`.
+1. in v1.28, `kubeadm upgrade` will respect the annotation and save it to local kubelet configuration or node
+   specific configuration `/var/lib/kubelet/kubeadm-config.yaml`. [TODO update according to the final decision]
 2. in v1.29, `kubeadm upgrade` will ignore the annotation.
 
 ### Test Plan

From 5d9d3dcc36a110dd13c1437d31363437fe56fc2b Mon Sep 17 00:00:00 2001
From: Paco Xu <paco.xu@daocloud.io>
Date: Mon, 3 Apr 2023 16:57:45 +0800
Subject: [PATCH 4/5] add feature gate for using

---
 .../sig-cluster-lifecycle/3929.yaml           |   3 -
 .../3929-no-cri-socket-annotation/README.md   | 380 +-----------------
 .../3929-no-cri-socket-annotation/kep.yaml    |   2 +-
 3 files changed, 21 insertions(+), 364 deletions(-)
 delete mode 100644 keps/prod-readiness/sig-cluster-lifecycle/3929.yaml

diff --git a/keps/prod-readiness/sig-cluster-lifecycle/3929.yaml b/keps/prod-readiness/sig-cluster-lifecycle/3929.yaml
deleted file mode 100644
index f9a687a8007..00000000000
--- a/keps/prod-readiness/sig-cluster-lifecycle/3929.yaml
+++ /dev/null
@@ -1,3 +0,0 @@
-kep-number: 3929
-alpha:
-  approver: "@"
diff --git a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
index c813da06c81..b22cf60978d 100644
--- a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
+++ b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
@@ -89,12 +89,22 @@ tags, and then generate with `hack/update-toc.sh`.
   - [Notes/Constraints/Caveats (Optional)](#notesconstraintscaveats-optional)
   - [Risks and Mitigations](#risks-and-mitigations)
 - [Design Details](#design-details)
+  - [init: upload a global kubelet configuration with cri socket](#init-upload-a-global-kubelet-configuration-with-cri-socket)
+  - [join: can override it using --config](#join-can-override-it-using---config)
+  - [upgrade: re-download global one, but should use local kubelet configuration firstly](#upgrade-re-download-global-one-but-should-use-local-kubelet-configuration-firstly)
+  - [Proposal 1: respect a list of configuration in local kubelet configuration, and in v1.27, CRI socket is the only one](#proposal-1-respect-a-list-of-configuration-in-local-kubelet-configuration-and-in-v127-cri-socket-is-the-only-one)
+  - [Proposal 2: introduce a <code>/var/lib/kubelet/kubeadm-config.yaml</code> to maintain node specific configuration](#proposal-2-introduce-a--to-maintain-node-specific-configuration)
+  - [old version handling](#old-version-handling)
   - [Test Plan](#test-plan)
-    - [Prerequisite testing updates](#prerequisite-testing-updates)
-    - [Unit tests](#unit-tests)
-    - [Integration tests](#integration-tests)
-    - [e2e tests](#e2e-tests)
+      - [Prerequisite testing updates](#prerequisite-testing-updates)
+      - [Unit tests](#unit-tests)
+      - [Integration tests](#integration-tests)
+      - [e2e tests](#e2e-tests)
   - [Graduation Criteria](#graduation-criteria)
+    - [Alpha](#alpha)
+    - [Beta](#beta)
+    - [GA](#ga)
+    - [Deprecation](#deprecation)
   - [Upgrade / Downgrade Strategy](#upgrade--downgrade-strategy)
   - [Version Skew Strategy](#version-skew-strategy)
 - [Production Readiness Review Questionnaire](#production-readiness-review-questionnaire)
@@ -221,7 +231,7 @@ cri socket in kubelet configuration.
 
 ### Proposal 1: respect a list of configuration in local kubelet configuration, and in v1.27, CRI socket is the only one
 
-During `kubeadm ugprade`, kubeadm will read the local kubelet configuration in `/var/lib/kubelet/config.yaml`.
+During `kubeadm upgrade`, kubeadm will read the local kubelet configuration in `/var/lib/kubelet/config.yaml`.
 kubeadm also download the kubelet configuration from configmap and replace the `containerRuntimeEndpoint` and
 `imageServiceEndpoint`(This maybe empty and I prefer to respect it as well) with the local configuration.
 
@@ -239,6 +249,11 @@ It is similar to `/var/lib/kubelet/kubeadm-flags.env`.
 KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=k8s.m.daocloud.io/pause:3.9"
 ```
 
+We may introduce a feature gate "KubeadmNodeSpecificConfig" to enable the use the `/var/lib/kubelet/kubeadm-config.yaml` here.
+
+- If the feature gate is disabled, use the cri socket annotation directly.
+- If the feature gate is enabled, `/var/lib/kubelet/kubeadm-config.yaml` will be created and the cri socket will be maintained in it.
+
 [To be discussed] Another proposal is using a strategy like `--patch`. A file like `/var/lib/kubelet/kubeadm-config.patch`
 or a `kubelet.yaml`/`config.ayml` file under `/var/lib/kubelet/patch/`. (This should be removed if we make a decision).
 
@@ -252,17 +267,6 @@ For old version cluster upgradation with the annotation, we will not touch the a
 
 ### Test Plan
 
-<!--
-**Note:** *Not required until targeted at a release.*
-The goal is to ensure that we don't accept enhancements with inadequate testing.
-
-All code is expected to have adequate tests (eventually with coverage
-expectations). Please adhere to the [Kubernetes testing guidelines][testing-guidelines]
-when drafting this test plan.
-
-[testing-guidelines]: https://git.k8s.io/community/contributors/devel/sig-testing/testing.md
--->
-
 [x] I/we understand the owners of the involved components may require updates to
 existing tests to make this code solid enough prior to committing the changes necessary
 to implement this enhancement.
@@ -274,58 +278,16 @@ Install/Join/Upgrade test in <https://testgrid.k8s.io/sig-cluster-lifecycle-kube
 
 ##### Prerequisite testing updates
 
-<!--
-Based on reviewers feedback describe what additional tests need to be added prior
-implementing this enhancement to ensure the enhancements have also solid foundations.
--->
-
 ##### Unit tests
 
-<!--
-In principle every added code should have complete unit test coverage, so providing
-the exact set of tests will not bring additional value.
-However, if complete unit test coverage is not possible, explain the reason of it
-together with explanation why this is acceptable.
--->
-
-<!--
-Additionally, for Alpha try to enumerate the core package you will be touching
-to implement this enhancement and provide the current unit coverage for those
-in the form of:
-- <package>: <date> - <current test coverage>
-The data can be easily read from:
-https://testgrid.k8s.io/sig-testing-canaries#ci-kubernetes-coverage-unit
-
-This can inform certain test coverage improvements that we want to do before
-extending the production code to implement this enhancement.
--->
-
 - `<package>`: `<date>` - `<test coverage>`
 
 ##### Integration tests
 
-<!--
-This question should be filled when targeting a release.
-For Alpha, describe what tests will be added to ensure proper quality of the enhancement.
-
-For Beta and GA, add links to added tests together with links to k8s-triage for those tests:
-https://storage.googleapis.com/k8s-triage/index.html
--->
-
 - <test>: <link to test coverage>
 
 ##### e2e tests
 
-<!--
-This question should be filled when targeting a release.
-For Alpha, describe what tests will be added to ensure proper quality of the enhancement.
-
-For Beta and GA, add links to added tests together with links to k8s-triage for those tests:
-https://storage.googleapis.com/k8s-triage/index.html
-
-We expect no non-infra related flakes in the last month as a GA graduation criteria.
--->
-
 - <test>: <link to test coverage>
 
 ### Graduation Criteria
@@ -363,61 +325,12 @@ See above.
 
 ### Version Skew Strategy
 
-<!--
-If applicable, how will the component handle version skew with other
-components? What are the guarantees? Make sure this is in the test plan.
-
-Consider the following in developing a version skew strategy for this
-enhancement:
-- Does this enhancement involve coordinating behavior in the control plane and
-  in the kubelet? How does an n-2 kubelet without this feature available behave
-  when this feature is used?
-- Will any other components on the node change? For example, changes to CSI,
-  CRI or CNI may require updating that component before the kubelet.
--->
-
 ## Production Readiness Review Questionnaire
 
-<!--
-
-Production readiness reviews are intended to ensure that features merging into
-Kubernetes are observable, scalable and supportable; can be safely operated in
-production environments, and can be disabled or rolled back in the event they
-cause increased failures in production. See more in the PRR KEP at
-https://git.k8s.io/enhancements/keps/sig-architecture/1194-prod-readiness.
-
-The production readiness review questionnaire must be completed and approved
-for the KEP to move to `implementable` status and be included in the release.
-
-In some cases, the questions below should also have answers in `kep.yaml`. This
-is to enable automation to verify the presence of the review, and to reduce review
-burden and latency.
-
-The KEP must have a approver from the
-[`prod-readiness-approvers`](http://git.k8s.io/enhancements/OWNERS_ALIASES)
-team. Please reach out on the
-[#prod-readiness](https://kubernetes.slack.com/archives/CPNHUMN74) channel if
-you need any help or guidance.
--->
-
 ### Feature Enablement and Rollback
 
-<!--
-This section must be completed when targeting alpha to a release.
--->
-
 ###### How can this feature be enabled / disabled in a live cluster?
 
-<!--
-Pick one of these and delete the rest.
-
-Documentation is available on [feature gate lifecycle] and expectations, as
-well as the [existing list] of feature gates.
-
-[feature gate lifecycle]: https://git.k8s.io/community/contributors/devel/sig-architecture/feature-gates.md
-[existing list]: https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/
--->
-
 - [ ] Feature gate (also fill in values in `kep.yaml`)
   - Feature gate name: No
   - Components depending on the feature gate:
@@ -430,108 +343,28 @@ well as the [existing list] of feature gates.
 
 ###### Does enabling the feature change any default behavior?
 
-<!--
-Any change of default behavior may be surprising to users or break existing
-automations, so be extremely careful here.
--->
-
 ###### Can the feature be disabled once it has been enabled (i.e. can we roll back the enablement)?
 
-<!--
-Describe the consequences on existing workloads (e.g., if this is a runtime
-feature, can it break the existing applications?).
-
-Feature gates are typically disabled by setting the flag to `false` and
-restarting the component. No other changes should be necessary to disable the
-feature.
-
-NOTE: Also set `disable-supported` to `true` or `false` in `kep.yaml`.
--->
-
 ###### What happens if we reenable the feature if it was previously rolled back?
 
 ###### Are there any tests for feature enablement/disablement?
 
-<!--
-The e2e framework does not currently support enabling or disabling feature
-gates. However, unit tests in each component dealing with managing data, created
-with and without the feature, are necessary. At the very least, think about
-conversion tests if API types are being modified.
-
-Additionally, for features that are introducing a new API field, unit tests that
-are exercising the `switch` of feature gate itself (what happens if I disable a
-feature gate after having objects written with the new field) are also critical.
-You can take a look at one potential example of such test in:
-https://github.com/kubernetes/kubernetes/pull/97058/files#diff-7826f7adbc1996a05ab52e3f5f02429e94b68ce6bce0dc534d1be636154fded3R246-R282
--->
-
 ### Rollout, Upgrade and Rollback Planning
 
-<!--
-This section must be completed when targeting beta to a release.
--->
-
 ###### How can a rollout or rollback fail? Can it impact already running workloads?
 
-<!--
-Try to be as paranoid as possible - e.g., what if some components will restart
-mid-rollout?
-
-Be sure to consider highly-available clusters, where, for example,
-feature flags will be enabled on some API servers and not others during the
-rollout. Similarly, consider large clusters and how enablement/disablement
-will rollout across nodes.
--->
-
 ###### What specific metrics should inform a rollback?
 
-<!--
-What signals should users be paying attention to when the feature is young
-that might indicate a serious problem?
--->
-
 ###### Were upgrade and rollback tested? Was the upgrade->downgrade->upgrade path tested?
 
-<!--
-Describe manual testing that was done and the outcomes.
-Longer term, we may want to require automated upgrade/rollback tests, but we
-are missing a bunch of machinery and tooling and can't do that now.
--->
-
 ###### Is the rollout accompanied by any deprecations and/or removals of features, APIs, fields of API types, flags, etc.?
 
-<!--
-Even if applying deprecation policies, they may still surprise some users.
--->
-
 ### Monitoring Requirements
 
-<!--
-This section must be completed when targeting beta to a release.
-
-For GA, this section is required: approvers should be able to confirm the
-previous answers based on experience in the field.
--->
-
 ###### How can an operator determine if the feature is in use by workloads?
 
-<!--
-Ideally, this should be a metric. Operations against the Kubernetes API (e.g.,
-checking if there are objects with field X set) may be a last resort. Avoid
-logs or events for this purpose.
--->
-
 ###### How can someone using this feature know that it is working for their instance?
 
-<!--
-For instance, if this is a pod-related feature, it should be possible to determine if the feature is functioning properly
-for each individual pod.
-Pick one more of these and delete the rest.
-Please describe all items visible to end users below with sufficient detail so that they can verify correct enablement
-and operation of this feature.
-Recall that end users cannot usually observe component logs or access metrics.
--->
-
 - [ ] Events
   - Event Reason:
 - [ ] API .status
@@ -542,215 +375,42 @@ Recall that end users cannot usually observe component logs or access metrics.
 
 ###### What are the reasonable SLOs (Service Level Objectives) for the enhancement?
 
-<!--
-This is your opportunity to define what "normal" quality of service looks like
-for a feature.
-
-It's impossible to provide comprehensive guidance, but at the very
-high level (needs more precise definitions) those may be things like:
-  - per-day percentage of API calls finishing with 5XX errors <= 1%
-  - 99% percentile over day of absolute value from (job creation time minus expected
-    job creation time) for cron job <= 10%
-  - 99.9% of /health requests per day finish with 200 code
-
-These goals will help you determine what you need to measure (SLIs) in the next
-question.
--->
-
 ###### What are the SLIs (Service Level Indicators) an operator can use to determine the health of the service?
 
-<!--
-Pick one more of these and delete the rest.
--->
-
-- [ ] Metrics
-  - Metric name:
-  - [Optional] Aggregation method:
-  - Components exposing the metric:
-- [ ] Other (treat as last resort)
-  - Details:
-
 ###### Are there any missing metrics that would be useful to have to improve observability of this feature?
 
-<!--
-Describe the metrics themselves and the reasons why they weren't added (e.g., cost,
-implementation difficulties, etc.).
--->
-
 ### Dependencies
 
-<!--
-This section must be completed when targeting beta to a release.
--->
-
 ###### Does this feature depend on any specific services running in the cluster?
 
-<!--
-Think about both cluster-level services (e.g. metrics-server) as well
-as node-level agents (e.g. specific version of CRI). Focus on external or
-optional services that are needed. For example, if this feature depends on
-a cloud provider API, or upon an external software-defined storage or network
-control plane.
-
-For each of these, fill in the following—thinking about running existing user workloads
-and creating new ones, as well as about cluster-level services (e.g. DNS):
-  - [Dependency name]
-    - Usage description:
-      - Impact of its outage on the feature:
-      - Impact of its degraded performance or high-error rates on the feature:
--->
-
 ### Scalability
 
-<!--
-For alpha, this section is encouraged: reviewers should consider these questions
-and attempt to answer them.
-
-For beta, this section is required: reviewers must answer these questions.
-
-For GA, this section is required: approvers should be able to confirm the
-previous answers based on experience in the field.
--->
-
 ###### Will enabling / using this feature result in any new API calls?
 
-<!--
-Describe them, providing:
-  - API call type (e.g. PATCH pods)
-  - estimated throughput
-  - originating component(s) (e.g. Kubelet, Feature-X-controller)
-Focusing mostly on:
-  - components listing and/or watching resources they didn't before
-  - API calls that may be triggered by changes of some Kubernetes resources
-    (e.g. update of object X triggers new updates of object Y)
-  - periodic API calls to reconcile state (e.g. periodic fetching state,
-    heartbeats, leader election, etc.)
--->
-
 ###### Will enabling / using this feature result in introducing new API types?
 
-<!--
-Describe them, providing:
-  - API type
-  - Supported number of objects per cluster
-  - Supported number of objects per namespace (for namespace-scoped objects)
--->
-
 ###### Will enabling / using this feature result in any new calls to the cloud provider?
 
-<!--
-Describe them, providing:
-  - Which API(s):
-  - Estimated increase:
--->
-
 ###### Will enabling / using this feature result in increasing size or count of the existing API objects?
 
-<!--
-Describe them, providing:
-  - API type(s):
-  - Estimated increase in size: (e.g., new annotation of size 32B)
-  - Estimated amount of new objects: (e.g., new Object X for every existing Pod)
--->
-
 ###### Will enabling / using this feature result in increasing time taken by any operations covered by existing SLIs/SLOs?
 
-<!--
-Look at the [existing SLIs/SLOs].
-
-Think about adding additional work or introducing new steps in between
-(e.g. need to do X to start a container), etc. Please describe the details.
-
-[existing SLIs/SLOs]: https://git.k8s.io/community/sig-scalability/slos/slos.md#kubernetes-slisslos
--->
-
 ###### Will enabling / using this feature result in non-negligible increase of resource usage (CPU, RAM, disk, IO, ...) in any components?
 
-<!--
-Things to keep in mind include: additional in-memory state, additional
-non-trivial computations, excessive access to disks (including increased log
-volume), significant amount of data sent and/or received over network, etc.
-This through this both in small and large cases, again with respect to the
-[supported limits].
-
-[supported limits]: https://git.k8s.io/community//sig-scalability/configs-and-limits/thresholds.md
--->
-
 ###### Can enabling / using this feature result in resource exhaustion of some node resources (PIDs, sockets, inodes, etc.)?
 
-<!--
-Focus not just on happy cases, but primarily on more pathological cases
-(e.g. probes taking a minute instead of milliseconds, failed pods consuming resources, etc.).
-If any of the resources can be exhausted, how this is mitigated with the existing limits
-(e.g. pods per node) or new limits added by this KEP?
-
-Are there any tests that were run/should be run to understand performance characteristics better
-and validate the declared limits?
--->
-
 ### Troubleshooting
 
-<!--
-This section must be completed when targeting beta to a release.
-
-For GA, this section is required: approvers should be able to confirm the
-previous answers based on experience in the field.
-
-The Troubleshooting section currently serves the `Playbook` role. We may consider
-splitting it into a dedicated `Playbook` document (potentially with some monitoring
-details). For now, we leave it here.
--->
-
 ###### How does this feature react if the API server and/or etcd is unavailable?
 
 ###### What are other known failure modes?
 
-<!--
-For each of them, fill in the following information by copying the below template:
-  - [Failure mode brief description]
-    - Detection: How can it be detected via metrics? Stated another way:
-      how can an operator troubleshoot without logging into a master or worker node?
-    - Mitigations: What can be done to stop the bleeding, especially for already
-      running user workloads?
-    - Diagnostics: What are the useful log messages and their required logging
-      levels that could help debug the issue?
-      Not required until feature graduated to beta.
-    - Testing: Are there any tests for failure mode? If not, describe why.
--->
-
 ###### What steps should be taken if SLOs are not being met to determine the problem?
 
 ## Implementation History
 
-<!--
-Major milestones in the lifecycle of a KEP should be tracked in this section.
-Major milestones might include:
-- the `Summary` and `Motivation` sections being merged, signaling SIG acceptance
-- the `Proposal` section being merged, signaling agreement on a proposed design
-- the date implementation started
-- the first Kubernetes release where an initial version of the KEP was available
-- the version of Kubernetes where the KEP graduated to general availability
-- when the KEP was retired or superseded
--->
-
 ## Drawbacks
 
-<!--
-Why should this KEP _not_ be implemented?
--->
-
 ## Alternatives
 
-<!--
-What other approaches did you consider, and why did you rule them out? These do
-not need to be as detailed as the proposal, but should include enough
-information to express the idea and why it was not acceptable.
--->
-
 ## Infrastructure Needed (Optional)
-
-<!--
-Use this section if you need things from the project/SIG. Examples include a
-new subproject, repos requested, or GitHub details. Listing these here allows a
-SIG to get the process for these resources started right away.
--->
diff --git a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml
index 403320c0338..d271c447c93 100644
--- a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml
+++ b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml
@@ -7,7 +7,7 @@ participating-sigs:
   - sig-cluster-lifecycle
 status: provisional
 creation-date: 2023-03-30
-last-updated: 2022-03-30
+last-updated: 2022-04-03
 reviewers:
   - "@neolit123"
 approvers:

From 1bb9613b2c3ef74f3402b1269aeb37b58fd46f63 Mon Sep 17 00:00:00 2001
From: Paco Xu <paco.xu@daocloud.io>
Date: Mon, 29 May 2023 12:24:09 +0800
Subject: [PATCH 5/5] use kubeadm-config-instance.yaml to save node specific
 configurations

---
 .../3929-no-cri-socket-annotation/README.md   | 59 ++++++++-----------
 .../3929-no-cri-socket-annotation/kep.yaml    |  3 +-
 2 files changed, 27 insertions(+), 35 deletions(-)

diff --git a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
index b22cf60978d..5f0c97f7dbc 100644
--- a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
+++ b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/README.md
@@ -92,9 +92,7 @@ tags, and then generate with `hack/update-toc.sh`.
   - [init: upload a global kubelet configuration with cri socket](#init-upload-a-global-kubelet-configuration-with-cri-socket)
   - [join: can override it using --config](#join-can-override-it-using---config)
   - [upgrade: re-download global one, but should use local kubelet configuration firstly](#upgrade-re-download-global-one-but-should-use-local-kubelet-configuration-firstly)
-  - [Proposal 1: respect a list of configuration in local kubelet configuration, and in v1.27, CRI socket is the only one](#proposal-1-respect-a-list-of-configuration-in-local-kubelet-configuration-and-in-v127-cri-socket-is-the-only-one)
-  - [Proposal 2: introduce a <code>/var/lib/kubelet/kubeadm-config.yaml</code> to maintain node specific configuration](#proposal-2-introduce-a--to-maintain-node-specific-configuration)
-  - [old version handling](#old-version-handling)
+  - [other proposal: respect a list of configuration in local kubelet configuration, and in v1.27, CRI socket is the only one](#other-proposal-respect-a-list-of-configuration-in-local-kubelet-configuration-and-in-v127-cri-socket-is-the-only-one)
   - [Test Plan](#test-plan)
       - [Prerequisite testing updates](#prerequisite-testing-updates)
       - [Unit tests](#unit-tests)
@@ -196,9 +194,15 @@ cri socket in kubelet configuration.
 
 ## Proposal
 
-1. init: upload a global kubelet configuration with cri socket
-2. join: can override it using --config
-3. upgrade: re-download global one, but should use local kubelet configuration firstly
+1. init: upload a global kubelet configuration with cri socket.
+   - the cri socket will take `--cri-socket` value and if the flag is empty, kubeadm will auto-detect it.
+   - After seting or auto-detecting, it will be set in the global kubelet configuration.
+2. join: it will use the global confugration.
+   - if it is not set in the global configuration, it will use `--cri-socket` value
+   - if it is still empty, kubeadm will auto-detect it.
+   - join will not change the global configuration, and if it is different with the global,
+     kubeadm will save it in `/var/lib/kubelet/kubeadm-config-instance.yaml`
+3. upgrade: re-download global one, but should use local kubelet configuration firstly in `kubeadm-config-instance.yaml`
 
 ### User Stories (Optional)
 
@@ -214,9 +218,11 @@ cri socket in kubelet configuration.
 
 ### init: upload a global kubelet configuration with cri socket
 
-- `kubeadm init` will not add the annotation to node.
+- `kubeadm init` will not add the annotation to node any more.
 - `kubeadm init` will check the customized `--config` at first and if no cri socket is set, it will
-  auto-detect it and save it global configuration and local as well.
+  auto-detect it and save it global configuration.
+  if `--cri-socket` is specified, we will use it in the local kubelet configuration and `kubeadm-config-instance.yaml`,
+  but it will not be saved to the global configuration.
 
 ### join: can override it using --config
 
@@ -224,12 +230,22 @@ cri socket in kubelet configuration.
 - `kubeadm join` will download the kubelet configuration from apiserver and the customized `--config`
   at first and auto-detect will work only if not set. Auto-detect may log a warning message if it may
   be misconfigured and log a general debug log if there is multi CRI-sockets.
+  if `--cri-socket` is specified, we will use it in the local kubelet configuration and `kubeadm-config-instance.yaml`,
+  but it will not be saved to the global configuration.
 
 ### upgrade: re-download global one, but should use local kubelet configuration firstly
 
 - `kubeadm upgrade` will download the kubelet configuration from apiserver and respect local one.
+- in v1.28-1.29, for backward compatibility, when `kubeadm upgrade apply`, we will read the `cri` annotation(if no annotation, we autodetect it)
+  and then patch it to the global configuration. `kubeadm upgrade node` is similar, and it will never change global configuration.
+- in v1.30+, `kubeadm upgrade apply` will not read the cri annotation any more.
+- in v1.28, for other nodes, `kubeadm upgrade node` will check if the cri annotation is diffent with the global setting.
+  if `cri-socket` is different, we will use it in the local kubelet configuration and `kubeadm-config-instance.yaml`,
+  but it will not be saved to the global configuration.
+- in v1.29 or later, `kubeadm upgrade node` will check `kubeadm-config-instance.yaml` at first and then check annoation like v1.28.
+- in v1.30+, `kubeadm upgrade node` will check `kubeadm-config-instance.yaml` and then global configuration only.
 
-### Proposal 1: respect a list of configuration in local kubelet configuration, and in v1.27, CRI socket is the only one
+### other proposal: respect a list of configuration in local kubelet configuration, and in v1.27, CRI socket is the only one
 
 During `kubeadm upgrade`, kubeadm will read the local kubelet configuration in `/var/lib/kubelet/config.yaml`.
 kubeadm also download the kubelet configuration from configmap and replace the `containerRuntimeEndpoint` and
@@ -240,31 +256,6 @@ A node-specific kubelet configuration list should be maintained in kubeadm code.
 - containerRuntimeEndpoint
 - imageServiceEndpoint
 
-### Proposal 2: introduce a `/var/lib/kubelet/kubeadm-config.yaml` to maintain node specific configuration
-
-We should introduce a `/var/lib/kubelet/kubeadm-config.yaml` to maintain node specific configuration.
-It is similar to `/var/lib/kubelet/kubeadm-flags.env`.
-
-```text
-KUBELET_KUBEADM_ARGS="--container-runtime-endpoint=unix:///var/run/containerd/containerd.sock --pod-infra-container-image=k8s.m.daocloud.io/pause:3.9"
-```
-
-We may introduce a feature gate "KubeadmNodeSpecificConfig" to enable the use the `/var/lib/kubelet/kubeadm-config.yaml` here.
-
-- If the feature gate is disabled, use the cri socket annotation directly.
-- If the feature gate is enabled, `/var/lib/kubelet/kubeadm-config.yaml` will be created and the cri socket will be maintained in it.
-
-[To be discussed] Another proposal is using a strategy like `--patch`. A file like `/var/lib/kubelet/kubeadm-config.patch`
-or a `kubelet.yaml`/`config.ayml` file under `/var/lib/kubelet/patch/`. (This should be removed if we make a decision).
-
-### old version handling
-
-For old version cluster upgradation with the annotation, we will not touch the annotation at first.
-
-1. in v1.28, `kubeadm upgrade` will respect the annotation and save it to local kubelet configuration or node
-   specific configuration `/var/lib/kubelet/kubeadm-config.yaml`. [TODO update according to the final decision]
-2. in v1.29, `kubeadm upgrade` will ignore the annotation.
-
 ### Test Plan
 
 [x] I/we understand the owners of the involved components may require updates to
diff --git a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml
index d271c447c93..212e63a744a 100644
--- a/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml
+++ b/keps/sig-cluster-lifecycle/kubeadm/3929-no-cri-socket-annotation/kep.yaml
@@ -7,9 +7,10 @@ participating-sigs:
   - sig-cluster-lifecycle
 status: provisional
 creation-date: 2023-03-30
-last-updated: 2022-04-03
+last-updated: 2022-05-29
 reviewers:
   - "@neolit123"
+  - "@chendave"
 approvers:
   - "@neolit123"
 latest-milestone: "0.0"