Skip to content

Commit

Permalink
✨ Add kubeadm v1beta4 types (#10709)
Browse files Browse the repository at this point in the history 
* Add kubeadm v1beta4 types

* cleanup conversions for kubeadm v1beta1, v1beta2, v1beta3

* Implement conversions for kubeadm v1beta4 types

* Handle Migration of ClusterConfiguration.APIServer.TimeoutForControlPlane to Init/JoinConfiguration.Timeouts.ControlPlaneComponentHealthCheck

* Handle Migration of JoinConfiguration.Discovery.Timeout to JoinConfiguration.Timeouts.TLSBootstrap

* Final alignment with kubeadm PR

* Address comments

* fix nit

---------

Co-authored-by: Stefan Bueringer <buringerst@vmware.com>
  • Loading branch information
@fabriziopandini @sbueringer
fabriziopandini and sbueringer committed Jun 17, 2024
1 parent bffacde commit a4c372d
Show file tree
Hide file tree
Showing 21 changed files with 3,390 additions and 224 deletions.
3 changes: 2 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -501,10 +501,11 @@ generate-go-conversions-kubeadm-bootstrap: $(CONVERSION_GEN) ## Generate convers
--extra-peer-dirs=sigs.k8s.io/cluster-api/internal/apis/core/v1alpha4 \
--output-file-base=zz_generated.conversion $(CONVERSION_GEN_OUTPUT_BASE) \
--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
$(MAKE) clean-generated-conversions SRC_DIRS="./bootstrap/kubeadm/types/upstreamv1beta2,./bootstrap/kubeadm/types/upstreamv1beta3"
$(MAKE) clean-generated-conversions SRC_DIRS="./bootstrap/kubeadm/types/upstreamv1beta2,./bootstrap/kubeadm/types/upstreamv1beta3,./bootstrap/kubeadm/types/upstreamv1beta4"
$(CONVERSION_GEN) \
--input-dirs=./bootstrap/kubeadm/types/upstreamv1beta2 \
--input-dirs=./bootstrap/kubeadm/types/upstreamv1beta3 \
--input-dirs=./bootstrap/kubeadm/types/upstreamv1beta4 \
--build-tag=ignore_autogenerated_kubeadm_types \
--output-file-base=zz_generated.conversion $(CONVERSION_GEN_OUTPUT_BASE) \
--go-header-file=./hack/boilerplate/boilerplate.generatego.txt
Expand Down
40 changes: 23 additions & 17 deletions bootstrap/kubeadm/internal/controllers/kubeadmconfig_controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -445,36 +445,38 @@ func (r *KubeadmConfigReconciler) handleClusterNotInitialized(ctx context.Contex
return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kubernetesVersion)
}

if scope.Config.Spec.InitConfiguration == nil {
scope.Config.Spec.InitConfiguration = &bootstrapv1.InitConfiguration{
if scope.Config.Spec.ClusterConfiguration == nil {
scope.Config.Spec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: "kubeadm.k8s.io/v1beta3",
Kind: "InitConfiguration",
APIVersion: "kubeadm.k8s.io/v1beta4",
Kind: "ClusterConfiguration",
},
}
}

initdata, err := kubeadmtypes.MarshalInitConfigurationForVersion(scope.Config.Spec.InitConfiguration, parsedVersion)
// injects into config.ClusterConfiguration values from top level object
r.reconcileTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config)

clusterdata, err := kubeadmtypes.MarshalClusterConfigurationForVersion(scope.Config.Spec.ClusterConfiguration, parsedVersion)
if err != nil {
scope.Error(err, "Failed to marshal init configuration")
scope.Error(err, "Failed to marshal cluster configuration")
return ctrl.Result{}, err
}

if scope.Config.Spec.ClusterConfiguration == nil {
scope.Config.Spec.ClusterConfiguration = &bootstrapv1.ClusterConfiguration{
if scope.Config.Spec.InitConfiguration == nil {
scope.Config.Spec.InitConfiguration = &bootstrapv1.InitConfiguration{
TypeMeta: metav1.TypeMeta{
APIVersion: "kubeadm.k8s.io/v1beta3",
Kind: "ClusterConfiguration",
APIVersion: "kubeadm.k8s.io/v1beta4",
Kind: "InitConfiguration",
},
}
}

// injects into config.ClusterConfiguration values from top level object
r.reconcileTopLevelObjectSettings(ctx, scope.Cluster, machine, scope.Config)

clusterdata, err := kubeadmtypes.MarshalClusterConfigurationForVersion(scope.Config.Spec.ClusterConfiguration, parsedVersion)
// NOTE: It is required to provide in input the ClusterConfiguration because clusterConfiguration.APIServer.TimeoutForControlPlane
// has been migrated to InitConfiguration in the kubeadm v1beta4 API version.
initdata, err := kubeadmtypes.MarshalInitConfigurationForVersion(scope.Config.Spec.ClusterConfiguration, scope.Config.Spec.InitConfiguration, parsedVersion)
if err != nil {
scope.Error(err, "Failed to marshal cluster configuration")
scope.Error(err, "Failed to marshal init configuration")
return ctrl.Result{}, err
}

Expand Down Expand Up @@ -602,7 +604,9 @@ func (r *KubeadmConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
joinConfiguration.NodeRegistration.Taints = append(joinConfiguration.NodeRegistration.Taints, clusterv1.NodeUninitializedTaint)
}

joinData, err := kubeadmtypes.MarshalJoinConfigurationForVersion(joinConfiguration, parsedVersion)
// NOTE: It is not required to provide in input ClusterConfiguration because only clusterConfiguration.APIServer.TimeoutForControlPlane
// has been migrated to JoinConfiguration in the kubeadm v1beta4 API version, and this field does not apply to workers.
joinData, err := kubeadmtypes.MarshalJoinConfigurationForVersion(nil, joinConfiguration, parsedVersion)
if err != nil {
scope.Error(err, "Failed to marshal join configuration")
return ctrl.Result{}, err
Expand Down Expand Up @@ -711,7 +715,9 @@ func (r *KubeadmConfigReconciler) joinControlplane(ctx context.Context, scope *S
return ctrl.Result{}, errors.Wrapf(err, "failed to parse kubernetes version %q", kubernetesVersion)
}

joinData, err := kubeadmtypes.MarshalJoinConfigurationForVersion(scope.Config.Spec.JoinConfiguration, parsedVersion)
// NOTE: It is required to provide in input the ClusterConfiguration because clusterConfiguration.APIServer.TimeoutForControlPlane
// has been migrated to JoinConfiguration in the kubeadm v1beta4 API version.
joinData, err := kubeadmtypes.MarshalJoinConfigurationForVersion(scope.Config.Spec.ClusterConfiguration, scope.Config.Spec.JoinConfiguration, parsedVersion)
if err != nil {
scope.Error(err, "Failed to marshal join configuration")
return ctrl.Result{}, err
Expand Down
32 changes: 14 additions & 18 deletions bootstrap/kubeadm/types/upstreamv1beta1/conversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,75 +23,71 @@ import (
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
)

// ConvertTo converts this ClusterConfiguration to the Hub version (v1alpha4).
func (src *ClusterConfiguration) ConvertTo(dstRaw conversion.Hub) error {
dst := dstRaw.(*bootstrapv1.ClusterConfiguration)
return Convert_upstreamv1beta1_ClusterConfiguration_To_v1beta1_ClusterConfiguration(src, dst, nil)
}

// ConvertFrom converts from the ClusterConfiguration Hub version (v1alpha4) to this version.
func (dst *ClusterConfiguration) ConvertFrom(srcRaw conversion.Hub) error {
src := srcRaw.(*bootstrapv1.ClusterConfiguration)
return Convert_v1beta1_ClusterConfiguration_To_upstreamv1beta1_ClusterConfiguration(src, dst, nil)
}

// ConvertTo converts this ClusterStatus to the Hub version (v1alpha4).
func (src *ClusterStatus) ConvertTo(dstRaw conversion.Hub) error {
dst := dstRaw.(*bootstrapv1.ClusterStatus)
return Convert_upstreamv1beta1_ClusterStatus_To_v1beta1_ClusterStatus(src, dst, nil)
}

// ConvertFrom converts from the ClusterStatus Hub version (v1alpha4) to this version.
func (dst *ClusterStatus) ConvertFrom(srcRaw conversion.Hub) error {
src := srcRaw.(*bootstrapv1.ClusterStatus)
return Convert_v1beta1_ClusterStatus_To_upstreamv1beta1_ClusterStatus(src, dst, nil)
}

// ConvertTo converts this InitConfiguration to the Hub version (v1alpha4).
func (src *InitConfiguration) ConvertTo(dstRaw conversion.Hub) error {
dst := dstRaw.(*bootstrapv1.InitConfiguration)
return Convert_upstreamv1beta1_InitConfiguration_To_v1beta1_InitConfiguration(src, dst, nil)
}

// ConvertFrom converts from the InitConfiguration Hub version (v1alpha4) to this version.
func (dst *InitConfiguration) ConvertFrom(srcRaw conversion.Hub) error {
src := srcRaw.(*bootstrapv1.InitConfiguration)
return Convert_v1beta1_InitConfiguration_To_upstreamv1beta1_InitConfiguration(src, dst, nil)
}

// ConvertTo converts this JoinConfiguration to the Hub version (v1alpha4).
func (src *JoinConfiguration) ConvertTo(dstRaw conversion.Hub) error {
dst := dstRaw.(*bootstrapv1.JoinConfiguration)
return Convert_upstreamv1beta1_JoinConfiguration_To_v1beta1_JoinConfiguration(src, dst, nil)
}

// ConvertFrom converts from the JoinConfiguration Hub version (v1alpha4) to this version.
func (dst *JoinConfiguration) ConvertFrom(srcRaw conversion.Hub) error {
src := srcRaw.(*bootstrapv1.JoinConfiguration)
return Convert_v1beta1_JoinConfiguration_To_upstreamv1beta1_JoinConfiguration(src, dst, nil)
}

func Convert_upstreamv1beta1_DNS_To_v1beta1_DNS(in *DNS, out *bootstrapv1.DNS, s apimachineryconversion.Scope) error {
// DNS.Type was removed in v1alpha4 because only CoreDNS is supported, dropping this info.
return autoConvert_upstreamv1beta1_DNS_To_v1beta1_DNS(in, out, s)
}
// Custom conversion from this API, kubeadm v1beta1, to the hub version, CABPK v1beta1.

func Convert_upstreamv1beta1_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in *ClusterConfiguration, out *bootstrapv1.ClusterConfiguration, s apimachineryconversion.Scope) error {
// ClusterConfiguration.UseHyperKubeImage was removed in kubeadm v1alpha4 API
// ClusterConfiguration.UseHyperKubeImage was removed in CABPK v1alpha4 API version, dropping this info (no issue, it was not used).
return autoConvert_upstreamv1beta1_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in, out, s)
}

func Convert_v1beta1_NodeRegistrationOptions_To_upstreamv1beta1_NodeRegistrationOptions(in *bootstrapv1.NodeRegistrationOptions, out *NodeRegistrationOptions, s apimachineryconversion.Scope) error {
// NodeRegistrationOptions.IgnorePreflightErrors does not exist in kubeadm v1beta1 API
return autoConvert_v1beta1_NodeRegistrationOptions_To_upstreamv1beta1_NodeRegistrationOptions(in, out, s)
func Convert_upstreamv1beta1_DNS_To_v1beta1_DNS(in *DNS, out *bootstrapv1.DNS, s apimachineryconversion.Scope) error {
// DNS.Type does not exist in CABPK v1beta1 version, because it always was CoreDNS.
return autoConvert_upstreamv1beta1_DNS_To_v1beta1_DNS(in, out, s)
}

// Custom conversion from the hub version, CABPK v1beta1, to this API, kubeadm v1beta1.

func Convert_v1beta1_InitConfiguration_To_upstreamv1beta1_InitConfiguration(in *bootstrapv1.InitConfiguration, out *InitConfiguration, s apimachineryconversion.Scope) error {
// InitConfiguration.Patches does not exist in kubeadm v1beta1 API
// InitConfiguration.SkipPhases and Patches does not exist in kubeadm v1beta1, dropping those info.
return autoConvert_v1beta1_InitConfiguration_To_upstreamv1beta1_InitConfiguration(in, out, s)
}

func Convert_v1beta1_JoinConfiguration_To_upstreamv1beta1_JoinConfiguration(in *bootstrapv1.JoinConfiguration, out *JoinConfiguration, s apimachineryconversion.Scope) error {
// JoinConfiguration.Patches does not exist in kubeadm v1beta1 API
// JoinConfiguration.SkipPhases and Patches does not exist in kubeadm v1beta1, dropping those info.
return autoConvert_v1beta1_JoinConfiguration_To_upstreamv1beta1_JoinConfiguration(in, out, s)
}

func Convert_v1beta1_NodeRegistrationOptions_To_upstreamv1beta1_NodeRegistrationOptions(in *bootstrapv1.NodeRegistrationOptions, out *NodeRegistrationOptions, s apimachineryconversion.Scope) error {
// NodeRegistrationOptions.IgnorePreflightErrors and ImagePullPolicy does not exist in kubeadm v1beta1, dropping those info.
return autoConvert_v1beta1_NodeRegistrationOptions_To_upstreamv1beta1_NodeRegistrationOptions(in, out, s)
}
58 changes: 27 additions & 31 deletions bootstrap/kubeadm/types/upstreamv1beta1/conversion_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,61 +60,57 @@ func TestFuzzyConversion(t *testing.T) {

func fuzzFuncs(_ runtimeserializer.CodecFactory) []interface{} {
return []interface{}{
dnsFuzzer,
clusterConfigurationFuzzer,
kubeadmNodeRegistrationOptionsFuzzer,
kubeadmInitConfigurationFuzzer,
kubeadmJoinConfigurationFuzzer,
dnsFuzzer,
bootstrapv1InitConfigurationFuzzer,
bootstrapv1JoinConfigurationFuzzer,
bootstrapv1NodeRegistrationOptionsFuzzer,
}
}

func dnsFuzzer(obj *DNS, c fuzz.Continue) {
c.FuzzNoCustom(obj)

// DNS.Type does not exists in v1alpha4, so setting it to empty string in order to avoid v1beta1 --> v1alpha4 --> v1beta1 round trip errors.
obj.Type = ""
}
// Custom fuzzers for kubeadm v1beta1 types.
// NOTES:
// - When fields do does not exist in cabpk v1beta1 types, pinning it to avoid kubeadm v1beta1 --> cabpk v1beta1 --> kubeadm v1beta1 round trip errors.

func clusterConfigurationFuzzer(obj *ClusterConfiguration, c fuzz.Continue) {
c.FuzzNoCustom(obj)

// ClusterConfiguration.UseHyperKubeImage has been removed in v1alpha4, so setting it to false in order to avoid v1beta1 --> v1alpha4 --> v1beta1 round trip errors.
obj.UseHyperKubeImage = false
}

func kubeadmNodeRegistrationOptionsFuzzer(obj *bootstrapv1.NodeRegistrationOptions, c fuzz.Continue) {
func dnsFuzzer(obj *DNS, c fuzz.Continue) {
c.FuzzNoCustom(obj)

// NodeRegistrationOptions.IgnorePreflightErrors does not exist in kubeadm v1beta1 API, so setting it to nil in order to avoid
// v1alpha4 --> v1beta1 -> v1alpha4 round trip errors.
obj.IgnorePreflightErrors = nil

// NodeRegistrationOptions.ImagePullPolicy does not exist in
// kubeadm v1beta1 API, so setting it to empty in order to
// avoid round trip errors.
obj.ImagePullPolicy = ""
obj.Type = ""
}

func kubeadmInitConfigurationFuzzer(obj *bootstrapv1.InitConfiguration, c fuzz.Continue) {
// Custom fuzzers for CABPK v1beta1 types.
// NOTES:
// - When fields do not exist in kubeadm v1beta1 types, pinning it to avoid cabpk v1beta1 --> kubeadm v1beta1 --> cabpk v1beta1 round trip errors.

func bootstrapv1InitConfigurationFuzzer(obj *bootstrapv1.InitConfiguration, c fuzz.Continue) {
c.FuzzNoCustom(obj)

// InitConfiguration.Patches does not exist in kubeadm v1beta1 API, so setting it to nil in order to avoid
// v1beta1 --> upstream v1beta1 -> v1beta1 round trip errors.
obj.Patches = nil

// InitConfiguration.SkipPhases does not exist in kubeadm v1beta1 API, so setting it to nil in order to avoid
// v1beta1 --> upstream v1beta1 -> v1beta1 round trip errors.
obj.SkipPhases = nil
}

func kubeadmJoinConfigurationFuzzer(obj *bootstrapv1.JoinConfiguration, c fuzz.Continue) {
func bootstrapv1JoinConfigurationFuzzer(obj *bootstrapv1.JoinConfiguration, c fuzz.Continue) {
c.FuzzNoCustom(obj)

// JoinConfiguration.Patches does not exist in kubeadm v1beta1 API, so setting it to nil in order to avoid
// v1beta1 --> upstream v1beta1 -> v1beta1 round trip errors.
// JoinConfiguration.Patches does not exist in kubeadm v1beta1 types, pinning it to avoid cabpk v1beta1 --> kubeadm v1beta1 --> cabpk v1beta1 round trip errors.
obj.Patches = nil

// JoinConfiguration.SkipPhases does not exist in kubeadm v1beta1 API, so setting it to nil in order to avoid
// v1beta1 --> upstream v1beta1 -> v1beta1 round trip errors.
// JoinConfiguration.SkipPhases does not exist in kubeadm v1beta1 types, pinning it to avoid cabpk v1beta1 --> kubeadm v1beta1 --> cabpk v1beta1 round trip errors.
obj.SkipPhases = nil
}

func bootstrapv1NodeRegistrationOptionsFuzzer(obj *bootstrapv1.NodeRegistrationOptions, c fuzz.Continue) {
c.FuzzNoCustom(obj)

// NodeRegistrationOptions.IgnorePreflightErrors does not exist in kubeadm v1beta1 types, pinning it to avoid cabpk v1beta1 --> kubeadm v1beta1 --> cabpk v1beta1 round trip errors.
obj.IgnorePreflightErrors = nil

// NodeRegistrationOptions.ImagePullPolicy does not exist in kubeadm v1beta1 types, pinning it to avoid cabpk v1beta1 --> kubeadm v1beta1 --> cabpk v1beta1 round trip errors.
obj.ImagePullPolicy = ""
}
33 changes: 18 additions & 15 deletions bootstrap/kubeadm/types/upstreamv1beta2/conversion.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,38 +63,41 @@ func (dst *JoinConfiguration) ConvertFrom(srcRaw conversion.Hub) error {
return Convert_v1beta1_JoinConfiguration_To_upstreamv1beta2_JoinConfiguration(src, dst, nil)
}

func Convert_upstreamv1beta2_InitConfiguration_To_v1beta1_InitConfiguration(in *InitConfiguration, out *bootstrapv1.InitConfiguration, s apimachineryconversion.Scope) error {
// InitConfiguration.CertificateKey exists in v1beta2 types but not in bootstrapv1.InitConfiguration (Cluster API does not uses automatic copy certs). Ignoring when converting.
return autoConvert_upstreamv1beta2_InitConfiguration_To_v1beta1_InitConfiguration(in, out, s)
}
// Custom conversion from this API, kubeadm v1beta2, to the hub version, CABPK v1beta1.

func Convert_upstreamv1beta2_JoinControlPlane_To_v1beta1_JoinControlPlane(in *JoinControlPlane, out *bootstrapv1.JoinControlPlane, s apimachineryconversion.Scope) error {
// JoinControlPlane.CertificateKey exists in v1beta2 types but not in bootstrapv1.JoinControlPlane (Cluster API does not uses automatic copy certs). Ignoring when converting.
return autoConvert_upstreamv1beta2_JoinControlPlane_To_v1beta1_JoinControlPlane(in, out, s)
func Convert_upstreamv1beta2_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in *ClusterConfiguration, out *bootstrapv1.ClusterConfiguration, s apimachineryconversion.Scope) error {
// ClusterConfiguration.UseHyperKubeImage was removed in CABPK v1alpha4 API version, dropping this info (no issue, it was not used).
return autoConvert_upstreamv1beta2_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in, out, s)
}

func Convert_upstreamv1beta2_DNS_To_v1beta1_DNS(in *DNS, out *bootstrapv1.DNS, s apimachineryconversion.Scope) error {
// DNS.Type was removed in v1alpha4 because only CoreDNS is supported, dropping this info.
// DNS.Type does not exist in CABPK v1beta1 version, because it always was CoreDNS.
return autoConvert_upstreamv1beta2_DNS_To_v1beta1_DNS(in, out, s)
}

func Convert_upstreamv1beta2_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in *ClusterConfiguration, out *bootstrapv1.ClusterConfiguration, s apimachineryconversion.Scope) error {
// ClusterConfiguration.UseHyperKubeImage was removed in kubeadm v1alpha4 API
return autoConvert_upstreamv1beta2_ClusterConfiguration_To_v1beta1_ClusterConfiguration(in, out, s)
func Convert_upstreamv1beta2_InitConfiguration_To_v1beta1_InitConfiguration(in *InitConfiguration, out *bootstrapv1.InitConfiguration, s apimachineryconversion.Scope) error {
// InitConfiguration.CertificateKey does not exist in CABPK v1beta1 version, because Cluster API does not use automatic copy certs.
return autoConvert_upstreamv1beta2_InitConfiguration_To_v1beta1_InitConfiguration(in, out, s)
}

func Convert_upstreamv1beta2_JoinControlPlane_To_v1beta1_JoinControlPlane(in *JoinControlPlane, out *bootstrapv1.JoinControlPlane, s apimachineryconversion.Scope) error {
// JoinControlPlane.CertificateKey does not exist in CABPK v1beta1 version, because Cluster API does not use automatic copy certs.
return autoConvert_upstreamv1beta2_JoinControlPlane_To_v1beta1_JoinControlPlane(in, out, s)
}

// Custom conversion from the hub version, CABPK v1beta1, to this API, kubeadm v1beta2.

func Convert_v1beta1_InitConfiguration_To_upstreamv1beta2_InitConfiguration(in *bootstrapv1.InitConfiguration, out *InitConfiguration, s apimachineryconversion.Scope) error {
// InitConfiguration.Patches does not exist in kubeadm v1beta2 API
// InitConfiguration.SkipPhases and Patches does not exist in kubeadm v1beta2, dropping those info.
return autoConvert_v1beta1_InitConfiguration_To_upstreamv1beta2_InitConfiguration(in, out, s)
}

func Convert_v1beta1_JoinConfiguration_To_upstreamv1beta2_JoinConfiguration(in *bootstrapv1.JoinConfiguration, out *JoinConfiguration, s apimachineryconversion.Scope) error {
// JoinConfiguration.Patches does not exist in kubeadm v1beta2 API
// JoinConfiguration.SkipPhases and Patches does not exist in kubeadm v1beta2, dropping those info.
return autoConvert_v1beta1_JoinConfiguration_To_upstreamv1beta2_JoinConfiguration(in, out, s)
}

func Convert_v1beta1_NodeRegistrationOptions_To_upstreamv1beta2_NodeRegistrationOptions(in *bootstrapv1.NodeRegistrationOptions, out *NodeRegistrationOptions, s apimachineryconversion.Scope) error {
// NodeRegistrationOptions.ImagePullPolicy does not exit in
// kubeadm v1beta2 API.
// NodeRegistrationOptions.IgnorePreflightErrors and ImagePullPolicy does not exist in kubeadm v1beta2, dropping those info.
return autoConvert_v1beta1_NodeRegistrationOptions_To_upstreamv1beta2_NodeRegistrationOptions(in, out, s)
}
Loading

0 comments on commit a4c372d

Please sign in to comment.