Update the support export tool to gather Patroni logs

docs/content/reference/pgo_support_export.md

@@ -101,6 +101,9 @@ Collecting networkpolicies...
 Collecting limitranges...
 Collecting events...
 Collecting Postgres logs...
+Collecting pgBackRest logs...
+Collecting Patroni logs...
+Collecting pgBackRest Repo Host logs...
 Collecting PostgresCluster pod logs...
 Collecting monitoring pod logs...
 Collecting operator pod logs...

internal/cmd/exec.go

@@ -79,6 +79,17 @@ func (exec Executor) listBackrestLogFiles() (string, string, error) {
 	return stdout.String(), stderr.String(), err
 }
 
+// listPatroniLogFiles returns the full path of Patroni log file.
+// These are the Patroni logs stored on the Postgres instance.
+func (exec Executor) listPatroniLogFiles() (string, string, error) {
+	var stdout, stderr bytes.Buffer
+
+	command := "ls -1dt pgdata/patroni/log/*"
+	err := exec(nil, &stdout, &stderr, "bash", "-ceu", "--", command)
+
+	return stdout.String(), stderr.String(), err
+}
+
 // listBackrestRepoHostLogFiles returns the full path of pgBackRest log files.
 // These are the pgBackRest logs stored on the repo host
 func (exec Executor) listBackrestRepoHostLogFiles() (string, string, error) {

internal/cmd/exec_test.go

@@ -79,6 +79,25 @@ func TestListPGLogFiles(t *testing.T) {
 
 }
 
+func TestListPatroniLogFiles(t *testing.T) {
+
+	t.Run("default", func(t *testing.T) {
+		expected := errors.New("pass-through")
+		exec := func(
+			stdin io.Reader, stdout, stderr io.Writer, command ...string,
+		) error {
+			assert.DeepEqual(t, command, []string{"bash", "-ceu", "--", "ls -1dt pgdata/patroni/log/*"})
+			assert.Assert(t, stdout != nil, "should capture stdout")
+			assert.Assert(t, stderr != nil, "should capture stderr")
+			return expected
+		}
+		_, _, err := Executor(exec).listPatroniLogFiles()
+		assert.ErrorContains(t, err, "pass-through")
+
+	})
+
+}
+
 func TestCatFile(t *testing.T) {
 
 	t.Run("default", func(t *testing.T) {

internal/cmd/export.go

@@ -282,6 +282,9 @@ Collecting networkpolicies...
 Collecting limitranges...
 Collecting events...
 Collecting Postgres logs...
+Collecting pgBackRest logs...
+Collecting Patroni logs...
+Collecting pgBackRest Repo Host logs...
 Collecting PostgresCluster pod logs...
 Collecting monitoring pod logs...
 Collecting operator pod logs...
@@ -471,6 +474,12 @@ Collecting PGO CLI logs...
 			writeInfo(cmd, fmt.Sprintf("Error gathering pgBackRest DB Hosts Logs: %s", err))
 		}
 
+		// Patroni Logs that are stored on the Postgres Instances
+		err = gatherPatroniLogs(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
+		if err != nil {
+			writeInfo(cmd, fmt.Sprintf("Error gathering Patroni Logs from Instance Pods: %s", err))
+		}
+
 		// All pgBackRest Logs on the Repo Host
 		err = gatherRepoHostLogs(ctx, clientset, restConfig, namespace, clusterName, tw, cmd)
 		if err != nil {
@@ -1308,6 +1317,113 @@ func gatherDbBackrestLogs(ctx context.Context,
 	return nil
 }
 
+// gatherPatroniLogs gathers all the file-based Patroni logs on the DB instance,
+// if configured. By default, these logs will be sent to stdout and captured as
+// Pod logs instead.
+func gatherPatroniLogs(ctx context.Context,
+	clientset *kubernetes.Clientset,
+	config *rest.Config,
+	namespace string,
+	clusterName string,
+	tw *tar.Writer,
+	cmd *cobra.Command,
+) error {
+	writeInfo(cmd, "Collecting Patroni logs...")
+
+	dbPods, err := clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
+		LabelSelector: util.DBInstanceLabels(clusterName),
+	})
+
+	if err != nil {
+		if apierrors.IsForbidden(err) {
+			writeInfo(cmd, err.Error())
+			return nil
+		}
+		return err
+	}
+
+	if len(dbPods.Items) == 0 {
+		writeInfo(cmd, "No database instance pod found for gathering logs")
+		return nil
+	}
+
+	writeDebug(cmd, fmt.Sprintf("Found %d Pods\n", len(dbPods.Items)))
+
+	podExec, err := util.NewPodExecutor(config)
+	if err != nil {
+		return err
+	}
+
+	for _, pod := range dbPods.Items {
+		writeDebug(cmd, fmt.Sprintf("Pod Name is %s\n", pod.Name))
+
+		exec := func(stdin io.Reader, stdout, stderr io.Writer, command ...string,
+		) error {
+			return podExec(namespace, pod.Name, util.ContainerDatabase,
+				stdin, stdout, stderr, command...)
+		}
+
+		// Get Patroni Log Files
+		stdout, stderr, err := Executor(exec).listPatroniLogFiles()
+
+		// Depending upon the list* function above:
+		// An error may happen when err is non-nil or stderr is non-empty.
+		// In both cases, we want to print helpful information and continue to the
+		// next iteration.
+		if err != nil || stderr != "" {
+
+			if apierrors.IsForbidden(err) {
+				writeInfo(cmd, err.Error())
+				return nil
+			}
+
+			writeDebug(cmd, "Error getting Patroni logs\n")
+
+			if err != nil {
+				writeDebug(cmd, fmt.Sprintf("%s\n", err.Error()))
+			}
+			if stderr != "" {
+				writeDebug(cmd, stderr)
+			}
+
+			if strings.Contains(stderr, "No such file or directory") {
+				writeDebug(cmd, "Cannot find any Patroni log files. This is acceptable in some configurations.\n")
+			}
+			continue
+		}
+
+		logFiles := strings.Split(strings.TrimSpace(stdout), "\n")
+		for _, logFile := range logFiles {
+			writeDebug(cmd, fmt.Sprintf("LOG FILE: %s\n", logFile))
+			var buf bytes.Buffer
+
+			stdout, stderr, err := Executor(exec).catFile(logFile)
+			if err != nil {
+				if apierrors.IsForbidden(err) {
+					writeInfo(cmd, err.Error())
+					// Continue and output errors for each log file
+					// Allow the user to see and address all issues at once
+					continue
+				}
+				return err
+			}
+
+			buf.Write([]byte(stdout))
+			if stderr != "" {
+				str := fmt.Sprintf("\nError returned: %s\n", stderr)
+				buf.Write([]byte(str))
+			}
+
+			path := clusterName + fmt.Sprintf("/pods/%s/", pod.Name) + logFile
+			if err := writeTar(tw, buf.Bytes(), path, cmd); err != nil {
+				return err
+			}
+		}
+
+	}
+	return nil
+}
+
 // gatherRepoHostLogs gathers all the file-based pgBackRest logs on the repo host.
 // There may not be any logs depending upon pgBackRest's log-level-file.
 func gatherRepoHostLogs(ctx context.Context,