Add Python Support (#31)

* Add Python Support

* Update docs

Author: edeNFed

.github/workflows/release.yml

@@ -36,6 +36,14 @@ jobs:
         password: ${{ secrets.DOCKER_HUB_PASSWORD }}
         repository: verizondigital/kubectl-flame
         tags: ${{ steps.vars.outputs.tag }}-bpf
+    - name: Build Python Docker Image
+      uses: docker/build-push-action@v1
+      with:
+        dockerfile: 'agent/docker/python/Dockerfile'
+        username: ${{ secrets.DOCKER_HUB_USER }}
+        password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+        repository: verizondigital/kubectl-flame
+        tags: ${{ steps.vars.outputs.tag }}-python
     - name: Setup Go
       uses: actions/setup-go@v1
       with:

.krew.yaml

@@ -9,7 +9,7 @@ spec:
   description: |
     Generate CPU flame graphs without restarting pods and with low overhead.
   caveats: |
-    Currently, only Java is supported.
+    Currently supported languages: Go, Java (any JVM based language) and Python
   platforms:
     - {{addURIAndSha "https://github.com/VerizonMedia/kubectl-flame/releases/download/{{ .TagName }}/kubectl-flame_{{ .TagName }}_darwin_x86_64.tar.gz" .TagName | indent 6 }}
       bin: kubectl-flame

README.md

@@ -15,7 +15,7 @@ Running `kubectlf-flame` does **not** require any modification to existing pods.
 - [License](#license)
 
 ## Requirements
-* Supported languages: Go, Java (any JVM based language)
+* Supported languages: Go, Java (any JVM based language) and Python
 * Kubernetes cluster that use Docker as the container runtime (tested on GKE, EKS and AKS)
 
 ## Usage
@@ -63,6 +63,7 @@ See the release page for the full list of pre-built assets.
 Under the hood `kubectl-flame` use [async-profiler](https://github.com/jvm-profiling-tools/async-profiler) in order to generate flame graphs for Java applications. 
 Interaction with the target JVM is done via a shared `/tmp` folder.
 Golang support is based on [ebpf profiling](https://en.wikipedia.org/wiki/Berkeley_Packet_Filter).
+Python support is based on [py-spy](https://github.com/benfred/py-spy).
 
 ## Contribute
 Please refer to [the contributing.md file](Contributing.md) for information about how to get involved. We welcome issues, questions, and pull requests.

agent/docker/python/Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.14-buster as agentbuild
+WORKDIR /go/src/github.com/VerizonMedia/kubectl-flame
+ADD . /go/src/github.com/VerizonMedia/kubectl-flame
+RUN go get -d -v ./...
+RUN cd agent && go build -o /go/bin/agent
+
+FROM alpine AS pyspybuild
+RUN apk add python3
+RUN echo 'manylinux1_compatible = True' > /usr/lib/python3.8/site-packages/_manylinux.py
+RUN pip3 install py-spy==0.4.0.dev1
+
+FROM alpine
+RUN mkdir /app
+COPY --from=agentbuild /go/bin/agent /app/agent
+COPY --from=pyspybuild /usr/bin/py-spy /app/py-spy
+
+CMD [ "/app/agent" ]

agent/profiler/python.go

@@ -0,0 +1,40 @@
+package profiler
+
+import (
+	"bytes"
+	"github.com/VerizonMedia/kubectl-flame/agent/details"
+	"github.com/VerizonMedia/kubectl-flame/agent/utils"
+	"os/exec"
+	"strconv"
+)
+
+const (
+	pySpyLocation        = "/app/py-spy"
+	pythonOutputFileName = "/tmp/python.svg"
+)
+
+type PythonProfiler struct{}
+
+func (p *PythonProfiler) SetUp(job *details.ProfilingJob) error {
+	return nil
+}
+
+func (p *PythonProfiler) Invoke(job *details.ProfilingJob) error {
+	pid, err := utils.FindRootProcessId(job)
+	if err != nil {
+		return err
+	}
+
+	duration := strconv.Itoa(int(job.Duration.Seconds()))
+	cmd := exec.Command(pySpyLocation, "record", "-p", pid, "-o", pythonOutputFileName, "-d", duration, "-s", "-t")
+	var out bytes.Buffer
+	var stderr bytes.Buffer
+	cmd.Stdout = &out
+	cmd.Stderr = &stderr
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+
+	return utils.PublishFlameGraph(pythonOutputFileName)
+}

agent/profiler/root.go

@@ -12,8 +12,9 @@ type FlameGraphProfiler interface {
 }
 
 var (
-	jvm = JvmProfiler{}
-	bpf = BpfProfiler{}
+	jvm    = JvmProfiler{}
+	bpf    = BpfProfiler{}
+	python = PythonProfiler{}
 )
 
 func ForLanguage(lang api.ProgrammingLanguage) (FlameGraphProfiler, error) {
@@ -22,6 +23,8 @@ func ForLanguage(lang api.ProgrammingLanguage) (FlameGraphProfiler, error) {
 		return &jvm, nil
 	case api.Go:
 		return &bpf, nil
+	case api.Python:
+		return &python, nil
 	default:
 		return nil, fmt.Errorf("could not find profiler for language %s", lang)
 	}

agent/utils/process.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"bufio"
 	"errors"
 	"github.com/VerizonMedia/kubectl-flame/agent/details"
 	"github.com/VerizonMedia/kubectl-flame/api"
@@ -13,7 +14,8 @@ import (
 
 var (
 	defaultProcessNames = map[api.ProgrammingLanguage]string{
-		api.Java: "java",
+		api.Java:   "java",
+		api.Python: "python",
 	}
 )
 
@@ -97,3 +99,98 @@ func FindProcessId(job *details.ProfilingJob) (string, error) {
 
 	return "", errors.New("could not find any process")
 }
+
+func FindRootProcessId(job *details.ProfilingJob) (string, error) {
+	name := getProcessName(job)
+	procsAndParents := make(map[string]string)
+	proc, err := os.Open("/proc")
+	if err != nil {
+		return "", err
+	}
+
+	defer proc.Close()
+
+	for {
+		dirs, err := proc.Readdir(15)
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return "", err
+		}
+
+		for _, di := range dirs {
+			if !di.IsDir() {
+				continue
+			}
+
+			dname := di.Name()
+			if dname[0] < '0' || dname[0] > '9' {
+				continue
+			}
+
+			mi, err := mountinfo.GetMountInfo(path.Join("/proc", dname, "mountinfo"))
+			if err != nil {
+				continue
+			}
+
+			for _, m := range mi {
+				root := m.Root
+				if strings.Contains(root, job.PodUID) &&
+					strings.Contains(root, job.ContainerName) {
+
+					exeName, err := os.Readlink(path.Join("/proc", dname, "exe"))
+					if err != nil {
+						continue
+					}
+
+					ppid, err := getProcessPPID(dname)
+					if err != nil {
+						return "", err
+					}
+
+					if name != "" {
+						// search by process name
+						if strings.Contains(exeName, name) {
+							procsAndParents[dname] = ppid
+						}
+					} else {
+						procsAndParents[dname] = ppid
+					}
+				}
+			}
+		}
+	}
+
+	return findRootProcess(procsAndParents)
+}
+
+func getProcessPPID(pid string) (string, error) {
+	ppidKey := "PPid"
+	statusFile, err := os.Open(path.Join("/proc", pid, "status"))
+	if err != nil {
+		return "", err
+	}
+
+	defer statusFile.Close()
+	scanner := bufio.NewScanner(statusFile)
+	for scanner.Scan() {
+		text := scanner.Text()
+		if strings.Contains(text, ppidKey) {
+			return strings.Fields(text)[1], nil
+		}
+	}
+
+	return "", errors.New("unable to get process ppid")
+}
+
+func findRootProcess(procsAndParents map[string]string) (string, error) {
+	for process, ppid := range procsAndParents {
+		if _, ok := procsAndParents[ppid]; !ok {
+			// Found process with ppid that is not in the same programming language - this is the root
+			return process, nil
+		}
+	}
+
+	return "", errors.New("could not find root process")
+}

api/langs.go

@@ -3,12 +3,13 @@ package api
 type ProgrammingLanguage string
 
 const (
-	Java ProgrammingLanguage = "java"
-	Go   ProgrammingLanguage = "go"
+	Java   ProgrammingLanguage = "java"
+	Go     ProgrammingLanguage = "go"
+	Python ProgrammingLanguage = "python"
 )
 
 var (
-	supportedLangs = []ProgrammingLanguage{Java, Go}
+	supportedLangs = []ProgrammingLanguage{Java, Go, Python}
 )
 
 func AvailableLanguages() []ProgrammingLanguage {

cli/cmd/data/target.go

@@ -18,7 +18,7 @@ type TargetDetails struct {
 	Alpine        bool
 	DryRun        bool
 	Image         string
-        DockerPath    string
+	DockerPath    string
 	Language      api.ProgrammingLanguage
 	Pgrep         string
 }

cli/cmd/kubernetes/job/python.go

@@ -0,0 +1,80 @@
+package job
+
+import (
+	"fmt"
+	"github.com/VerizonMedia/kubectl-flame/cli/cmd/data"
+	"github.com/VerizonMedia/kubectl-flame/cli/cmd/version"
+	batchv1 "k8s.io/api/batch/v1"
+	apiv1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/uuid"
+)
+
+type pythonCreator struct{}
+
+func (p *pythonCreator) create(targetPod *apiv1.Pod, targetDetails *data.TargetDetails) (string, *batchv1.Job) {
+	id := string(uuid.NewUUID())
+	var imageName string
+	if targetDetails.Image != "" {
+		imageName = targetDetails.Image
+	} else {
+		imageName = fmt.Sprintf("%s:%s-python", baseImageName, version.GetCurrent())
+	}
+
+	commonMeta := metav1.ObjectMeta{
+		Name:      fmt.Sprintf("kubectl-flame-%s", id),
+		Namespace: targetDetails.Namespace,
+		Labels: map[string]string{
+			"kubectl-flame/id": id,
+		},
+		Annotations: map[string]string{
+			"sidecar.istio.io/inject": "false",
+		},
+	}
+
+	job := &batchv1.Job{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Job",
+			APIVersion: "batch/v1",
+		},
+		ObjectMeta: commonMeta,
+		Spec: batchv1.JobSpec{
+			Parallelism:             int32Ptr(1),
+			Completions:             int32Ptr(1),
+			TTLSecondsAfterFinished: int32Ptr(5),
+			Template: apiv1.PodTemplateSpec{
+				ObjectMeta: commonMeta,
+				Spec: apiv1.PodSpec{
+					HostPID:        true,
+					InitContainers: nil,
+					Containers: []apiv1.Container{
+						{
+							ImagePullPolicy: apiv1.PullAlways,
+							Name:            ContainerName,
+							Image:           imageName,
+							Command:         []string{"/app/agent"},
+							Args: []string{id,
+								string(targetPod.UID),
+								targetDetails.ContainerName,
+								targetDetails.ContainerId,
+								targetDetails.Duration.String(),
+								string(targetDetails.Language),
+								targetDetails.Pgrep,
+							},
+							SecurityContext: &apiv1.SecurityContext{
+								Privileged: boolPtr(true),
+								Capabilities: &apiv1.Capabilities{
+									Add: []apiv1.Capability{"SYS_PTRACE"},
+								},
+							},
+						},
+					},
+					RestartPolicy: "Never",
+					NodeName:      targetPod.Spec.NodeName,
+				},
+			},
+		},
+	}
+
+	return id, job
+}

cli/cmd/kubernetes/job/root.go

@@ -16,8 +16,9 @@ const (
 )
 
 var (
-	jvm = jvmCreator{}
-	bpf = bpfCreator{}
+	jvm    = jvmCreator{}
+	bpf    = bpfCreator{}
+	python = pythonCreator{}
 )
 
 type creator interface {
@@ -30,6 +31,8 @@ func Create(targetPod *apiv1.Pod, targetDetails *data.TargetDetails) (string, *b
 		return jvm.create(targetPod, targetDetails)
 	case api.Go:
 		return bpf.create(targetPod, targetDetails)
+	case api.Python:
+		return python.create(targetPod, targetDetails)
 	}
 
 	// Should not happen