eBay Cloud Network团队解决Kubernetes中CPU过载问题的案例分享

收到告警电话，扩容CPU

Unset

10-->20-->40 CPU 扩容的过程

pprof 排查CPU 使用率罪魁祸首

Unset

func (p *TLBProvider) lockAllocationForPods(pods []v1.Pod, service *v1.Service) error {

     if pods == nil {

             return nil

      }

      for _, pod := range pods {

         // Check if allocation already exists for this pod

         labelSelector := labels.Set{

           PodNodeNameLabel:  pod.Spec.NodeName,

           PodNamespaceLabel: pod.Namespace,

           PodNameLabel:      pod.Name,

           PodUIDLabel:       string(pod.UID),

         }.AsSelector()

         allocations, err := p.allocationLister.List(labelSelector)

client-go 中的cache提供的源码分析

Unset

func ListAll(store Store, selector labels.Selector, appendFn AppendFunc) error {

      selectAll := selector.Empty()

      for _, m := range store.List() {

          if selectAll {

               // Avoid computing labels of the objects to speed up common flows

              // of listing all objects.

              appendFn(m)

              continue

           }

           metadata, err := meta.Accessor(m)

           if err != nil {

               return err

           }

           if selector.Matches(labels.Set(metadata.GetLabels())) {

               appendFn(m)

            }

         }

         return nil

}

Unset

// Store is a generic object storage interface. Reflector knows how to watch a server

// and update a store. A generic store is provided, which allows Reflector to be used

// as a local caching system, and an LRU store, which allows Reflector to work like a

// queue of items yet to be processed.

//

// Store makes no assumptions about stored object identity; it is the responsibility

// of a Store implementation to provide a mechanism to correctly key objects and to

// define the contract for obtaining objects by some arbitrary key type.

type Store interface {

        Add(obj interface{}) error

        Update(obj interface{}) error

        Delete(obj interface{}) error

        List() []interface{}

        ListKeys() []string

        Get(obj interface{}) (item interface{}, exists bool, err error)

        GetByKey(key string) (item interface{}, exists bool, err error)

         // Replace will delete the contents of the store, using instead the

         // given list. Store takes ownership of the list, you should not reference

         // it after calling this function.

        Replace([]interface{}, string) error

        Resync() error

}

分析CPU过载原因

Unset

(base)  ~/ kubectl get allocation -A | wc -l

  145457  

通过比较Pod和allocation对象，我发现它们之间有一个交叉字段，即IP。因此，可以通过IP将这两个对象关联起来。由于 client-go提供了 添加了自定义 AddIndexers的功能，具体可查看An introduction to Go Kubernetes' informers 我们可以通过自定义Indexers来加快访问速度，比如下面是一个通过pod的IP而不是name和nameSpace来获取pod的例子。

给informeer添加indexer：

通过给pod的IP来获取pod

Unset

items, err := podsInformer.GetIndexer().ByIndex(ByIP, ip)

实现IpamIndexByIPFunc

Unset

const (

         IpamIndexByIP        = "IpamIndexByIP"

         Slash32SubnetSize = 32

)

func IpamIndexByIPFunc(obj interface{}) ([]string, error) {

          alloc, ok := obj.(*ipamv1.Allocation)

          if !ok {

                 return nil, fmt.Errorf("object is not a Allocation")

          }

          // Extract the IP addresses from the ipamv1.Allocation and return them as a list of strings.

          var ipList []string

          for _, subNet := range alloc.Status.Subnets {

                  if subNet != "" {

                       ip, ipNet, err := net.ParseCIDR(subNet)

                       if err != nil {

return nil, fmt.Errorf("failed to parse subnet %s: %s", subNet, err.Error())

}

                        subnetSize, _ := ipNet.Mask.Size()

                        if subnetSize == Slash32SubnetSize {

                               ipList = append(ipList, ip.String())

                        }

                    }

            }

            return ipList, nil

}

Unset

func (p *TLBProvider) lockAllocationForPods(pods []v1.Pod, service *v1.Service) error {

                       if pods == nil {

                              return nil

                       }

                       for _, pod := range pods {

alloc:=allocationInformer.Informer().GetIndexer().ByIndex(common.IpamIndexByIP, pod.Status.PodIP)

最终效果对比

本文详细介绍了Kubernetes中处理CPU过载的问题排查和解决过程，包括 CPU 扩容、分析 client-go 的 ListAll() 函数以及解决方案的实施。通过自定义索引器。团队成功提高了性能，将 CPU 使用率从高峰值 40 降低到稳定的 1~4 左右，同时没有带来额外的内存消耗。这个案例为类似问题的处理提供了有价值的参考。