Skip to content
This repository has been archived by the owner on Sep 24, 2021. It is now read-only.

Commit

Permalink
Merge pull request #162 from heartlock/support-portmapping
Browse files Browse the repository at this point in the history 
Support portmapping
  • Loading branch information
@feiskyer
feiskyer authored Jun 9, 2017
2 parents 222e798 + 6f71d9e commit c8eacdb
Show file tree
Hide file tree
Showing 6 changed files with 155 additions and 50 deletions.
45 changes: 28 additions & 17 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,29 +52,40 @@ systemctl restart hyperd
Setup CNI networking using bridge plugin

```sh
$ sudo mkdir -p /etc/cni/net.d /opt/cni/bin
$ curl -sSL https://github.com/containernetworking/cni/releases/download/v0.5.2/cni-amd64-v0.5.2.tgz -o cni-amd64-v0.5.2.tgz
$ sudo tar zxvf cni-amd64-v0.5.2.tgz -C /opt/cni/bin && rm -f cni-amd64-v0.5.2.tgz
$ sudo sh -c 'cat >/etc/cni/net.d/10-mynet.conf <<-EOF
$ sudo mkdir -p /etc/cni/net.d /opt/cni/bin
$ git clone https://github.com/containernetworking/plugins $GOPATH/src/github.com/containernetworking/plugins
$ cd $GOPATH/src/github.com/containernetworking/plugins
$ ./build.sh
$ sudo cp bin/* /opt/cni/bin/
$ sudo sh -c 'cat >/etc/cni/net.d/10-mynet.conflist <<-EOF
{
"cniVersion": "0.3.0",
"cniVersion": "0.3.1",
"name": "mynet",
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.10.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
"plugins": [
{
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.30.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true},
"snat": true
}
]
}
EOF'
$ sudo sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<-EOF
{
"cniVersion": "0.3.0",
"cniVersion": "0.3.1",
"type": "loopback"
}
EOF'
Expand Down
48 changes: 30 additions & 18 deletions hack/test-e2e.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,32 +88,44 @@ function install_remote_hyperd() {
function configure_cni() {
# install cni
sudo mkdir -p /etc/cni/net.d /opt/cni/bin
curl -sSL https://github.com/containernetworking/cni/releases/download/${CNI_VERSION}/cni-amd64-${CNI_VERSION}.tgz -o cni.tgz
sudo tar zxvf cni.tgz -C /opt/cni/bin
rm -f cni.tgz

git clone https://github.com/containernetworking/plugins $GOPATH/src/github.com/containernetworking/plugins
cd $GOPATH/src/github.com/containernetworking/plugins

./build.sh
sudo cp bin/* /opt/cni/bin/

# create network configure file
sudo sh -c 'cat >/etc/cni/net.d/10-mynet.conf <<-EOF
sudo sh -c 'cat >/etc/cni/net.d/10-mynet.conflist <<-EOF
{
"cniVersion": "0.3.0",
"cniVersion": "0.3.1",
"name": "mynet",
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.30.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
"plugins": [
{
"type": "bridge",
"bridge": "cni0",
"isGateway": true,
"ipMasq": true,
"ipam": {
"type": "host-local",
"subnet": "10.30.0.0/16",
"routes": [
{ "dst": "0.0.0.0/0" }
]
}
},
{
"type": "portmap",
"capabilities": {"portMappings": true},
"snat": true
}
]
}
EOF'

sudo sh -c 'cat >/etc/cni/net.d/99-loopback.conf <<-EOF
{
"cniVersion": "0.3.0",
"cniVersion": "0.3.1",
"type": "loopback"
}
EOF'
Expand All @@ -124,7 +136,7 @@ function test_cri() {
go get github.com/kubernetes-incubator/cri-tools/cmd/critest

# run critest
sudo env PATH=$PATH:$GOPATH/bin GOPATH=$GOPATH critest -r=/var/run/frakti.sock --focus="Conformance" --skip="container port" validation
sudo env PATH=$PATH:$GOPATH/bin GOPATH=$GOPATH critest -r=/var/run/frakti.sock --focus="Conformance" validation
}

FRAKTI_LISTEN_ADDR=${FRAKTI_LISTEN_ADDR:-/var/run/frakti.sock}
Expand Down
46 changes: 46 additions & 0 deletions pkg/hyper/network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,19 @@ import (
cnitypes "github.com/containernetworking/cni/pkg/types"
"github.com/containernetworking/cni/pkg/types/current"
"github.com/vishvananda/netlink"
"k8s.io/kubernetes/pkg/api/v1"
"k8s.io/kubernetes/pkg/kubelet/network/hostport"
)

// cniPortMapping maps to the standard CNI portmapping Capability
// see: https://github.com/containernetworking/cni/blob/master/CONVENTIONS.md
type cniPortMapping struct {
HostPort int32 `json:"hostPort"`
ContainerPort int32 `json:"containerPort"`
Protocol string `json:"protocol"`
HostIP string `json:"hostIP"`
}

// The network information needed to create HyperContainer
// network device from CNI Result
type NetworkInfo struct {
Expand All @@ -35,6 +46,41 @@ type NetworkInfo struct {
BridgeName string
}

func (h *Runtime) GetPodPortMappings(podID string) ([]*hostport.PortMapping, error) {
// TODO: get portmappings from docker labels for backward compatibility
checkpoint, err := h.checkpointHandler.GetCheckpoint(podID)
if err != nil {
return nil, err

}

portMappings := make([]*hostport.PortMapping, 0, len(checkpoint.Data.PortMappings))
for _, pm := range checkpoint.Data.PortMappings {
proto := toAPIProtocol(*pm.Protocol)
portMappings = append(portMappings, &hostport.PortMapping{
HostPort: *pm.HostPort,
ContainerPort: *pm.ContainerPort,
Protocol: proto,
})

}
return portMappings, nil

}

func toAPIProtocol(protocol Protocol) v1.Protocol {
switch protocol {
case ProtocolTCP:
return v1.ProtocolTCP
case ProtocolUDP:
return v1.ProtocolUDP

}
glog.Warningf("Unknown protocol %q: defaulting to TCP", protocol)
return v1.ProtocolTCP

}

func convertCNIResult2NetworkInfo(result cnitypes.Result) *NetworkInfo {
ret := &NetworkInfo{}

Expand Down
21 changes: 12 additions & 9 deletions pkg/hyper/ocicni/ocicni.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ import (
"github.com/containernetworking/cni/libcni"
cnitypes "github.com/containernetworking/cni/pkg/types"
"github.com/golang/glog"
kubeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
)

// TODO: upgrade CNI Plugin to its stable realease
Expand Down Expand Up @@ -155,12 +156,12 @@ func (plugin *cniNetworkPlugin) Name() string {
return CNIPluginName
}

func (plugin *cniNetworkPlugin) SetUpPod(podNetnsPath string, podID string) (cnitypes.Result, error) {
func (plugin *cniNetworkPlugin) SetUpPod(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, annotations map[string]string, capabilities map[string]interface{}) (cnitypes.Result, error) {
if err := plugin.checkInitialized(); err != nil {
return nil, err
}

res, err := plugin.getDefaultNetwork().addToNetwork(podNetnsPath, podID)
res, err := plugin.getDefaultNetwork().addToNetwork(podNetnsPath, podID, metadata, capabilities)
if err != nil {
glog.Errorf("Error while adding to cni network: %s", err)
return nil, err
Expand All @@ -170,16 +171,16 @@ func (plugin *cniNetworkPlugin) SetUpPod(podNetnsPath string, podID string) (cni
return res, nil
}

func (plugin *cniNetworkPlugin) TearDownPod(podNetnsPath string, podID string) error {
func (plugin *cniNetworkPlugin) TearDownPod(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, annotations map[string]string, capabilities map[string]interface{}) error {
if err := plugin.checkInitialized(); err != nil {
return err
}

return plugin.getDefaultNetwork().deleteFromNetwork(podNetnsPath, podID)
return plugin.getDefaultNetwork().deleteFromNetwork(podNetnsPath, podID, metadata, capabilities)
}

func (network *cniNetwork) addToNetwork(podNetnsPath string, podID string) (cnitypes.Result, error) {
rt, err := buildCNIRuntimeConf(podNetnsPath, podID)
func (network *cniNetwork) addToNetwork(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, capabilities map[string]interface{}) (cnitypes.Result, error) {
rt, err := buildCNIRuntimeConf(podNetnsPath, podID, metadata, capabilities)
if err != nil {
glog.Errorf("Pod: %s, Netns: %s, Error adding network: %v", podID, podNetnsPath, err)
return nil, err
Expand All @@ -196,8 +197,8 @@ func (network *cniNetwork) addToNetwork(podNetnsPath string, podID string) (cnit
return res, nil
}

func (network *cniNetwork) deleteFromNetwork(podNetnsPath string, podID string) error {
rt, err := buildCNIRuntimeConf(podNetnsPath, podID)
func (network *cniNetwork) deleteFromNetwork(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, capabilities map[string]interface{}) error {
rt, err := buildCNIRuntimeConf(podNetnsPath, podID, metadata, capabilities)
if err != nil {
glog.Errorf("Pod: %s, Netns: %s, Error deleting network: %v", podID, podNetnsPath, err)
return err
Expand All @@ -214,7 +215,7 @@ func (network *cniNetwork) deleteFromNetwork(podNetnsPath string, podID string)
return nil
}

func buildCNIRuntimeConf(podNetnsPath string, podID string) (*libcni.RuntimeConf, error) {
func buildCNIRuntimeConf(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, capabilities map[string]interface{}) (*libcni.RuntimeConf, error) {
glog.V(4).Infof("Got netns path %v", podNetnsPath)
glog.V(4).Infof("Using netns path %v", podNetnsPath)

Expand All @@ -225,7 +226,9 @@ func buildCNIRuntimeConf(podNetnsPath string, podID string) (*libcni.RuntimeConf
Args: [][2]string{
{"IgnoreUnknown", "1"},
{"K8S_POD_NAME", podID},
{"K8S_POD_NAMESPACE", metadata.GetNamespace()},
},
CapabilityArgs: capabilities,
}

return rt, nil
Expand Down
5 changes: 3 additions & 2 deletions pkg/hyper/ocicni/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package ocicni

import (
cnitypes "github.com/containernetworking/cni/pkg/types"
kubeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1"
)

const (
Expand All @@ -34,10 +35,10 @@ type CNIPlugin interface {
Name() string

// SetUpPod is the method called when the pod is created
SetUpPod(podNetnsPath string, podID string) (cnitypes.Result, error)
SetUpPod(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, annotations map[string]string, capabilities map[string]interface{}) (cnitypes.Result, error)

// TearDownPod is the method called before pod stopped
TearDownPod(podNetnsPath string, podID string) error
TearDownPod(podNetnsPath string, podID string, metadata *kubeapi.PodSandboxMetadata, annotations map[string]string, capabilities map[string]interface{}) error

// NetworkStatus returns error if the network plugin is in error state
Status() error
Expand Down
40 changes: 36 additions & 4 deletions pkg/hyper/sandbox.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,22 @@ func (h *Runtime) RunPodSandbox(config *kubeapi.PodSandboxConfig) (string, error
userpod.Labels["NETNS"] = netNsPath

// Setup the network
portMappings := config.GetPortMappings()
portMappingsParam := make([]cniPortMapping, 0, len(portMappings))
for _, p := range portMappings {
protocol := kubeapi.Protocol_name[int32(p.Protocol)]
portMappingsParam = append(portMappingsParam, cniPortMapping{
HostPort: p.HostPort,
ContainerPort: p.ContainerPort,
Protocol: strings.ToLower(protocol),
HostIP: p.HostIp,
})
}
capabilities := map[string]interface{}{
"portMappings": portMappingsParam,
}
podId := userpod.Id
result, err := h.netPlugin.SetUpPod(netNsPath, podId)
result, err := h.netPlugin.SetUpPod(netNsPath, podId, config.GetMetadata(), config.GetAnnotations(), capabilities)
if err != nil {
glog.Errorf("Setup network for sandbox %q by cni plugin failed: %v", config.String(), err)
}
Expand Down Expand Up @@ -93,7 +107,7 @@ func (h *Runtime) RunPodSandbox(config *kubeapi.PodSandboxConfig) (string, error
glog.Warningf("Remove pod %q failed: %v", removeError)
}
// destroy the network namespace
if tearError := h.netPlugin.TearDownPod(netNsPath, podID); tearError != nil {
if tearError := h.netPlugin.TearDownPod(netNsPath, podID, config.GetMetadata(), config.GetAnnotations(), capabilities); tearError != nil {
glog.Errorf("Destroy pod %s network namespace failed: %v", podID, tearError)
}
unix.Unmount(netNsPath, unix.MNT_DETACH)
Expand Down Expand Up @@ -190,12 +204,30 @@ func (h *Runtime) StopPodSandbox(podSandboxID string) error {
netNsPath = checkpoint.NetNsPath
}

// Get portMappings from checkpoint
portMappings, err := h.GetPodPortMappings(podSandboxID)
if err != nil {
return fmt.Errorf("could not retrieve port mappings: %v", err)
}
portMappingsParam := make([]cniPortMapping, 0, len(portMappings))
for _, p := range portMappings {
portMappingsParam = append(portMappingsParam, cniPortMapping{
HostPort: p.HostPort,
ContainerPort: p.ContainerPort,
Protocol: strings.ToLower(string(p.Protocol)),
HostIP: p.HostIP,
})
}
capabilities := map[string]interface{}{
"portMappings": portMappingsParam,
}

code, cause, err := h.client.StopPod(podSandboxID)
if err != nil {
glog.Errorf("Stop pod %s failed, code: %d, cause: %s, error: %v", podSandboxID, code, cause, err)
if isPodNotFoundError(err, podSandboxID) {
// destroy the network namespace
err = h.netPlugin.TearDownPod(netNsPath, podSandboxID)
err = h.netPlugin.TearDownPod(netNsPath, podSandboxID, status.GetMetadata(), status.GetAnnotations(), capabilities)
if err != nil {
glog.Errorf("Destroy pod %s network namespace failed: %v", podSandboxID, err)
}
Expand All @@ -213,7 +245,7 @@ func (h *Runtime) StopPodSandbox(podSandboxID string) error {
}

// destroy the network namespace
err = h.netPlugin.TearDownPod(netNsPath, podSandboxID)
err = h.netPlugin.TearDownPod(netNsPath, podSandboxID, status.GetMetadata(), status.GetAnnotations(), capabilities)
if err != nil {
glog.Errorf("Destroy pod %s network namespace failed: %v", podSandboxID, err)
}
Expand Down

0 comments on commit c8eacdb

Please sign in to comment.