exclude_paths_glob_patterns does not work

[mindrunner]

I am using vector to push k8s logs to logtail. Some of my pods are very verbose and I want to exclude them by using exclude_paths_glob_patterns. However, I can still see logs of excluded pods being pushed to logtail. I also tried annotating pods with vector.dev/exclude: "true". That didn't work either. What am I missing here?

customConfig:
    data_dir: /vector-data-dir
    api:
      enabled: true
      address: 127.0.0.1:8686
      playground: false
    sources:
      kubernetes_logs:
        type: kubernetes_logs
        exclude_paths_glob_patterns: 
          - "**/dockermailserver/**"
          - "**/prep-config/**"
          - "**/timescaledb/**"
          - "**/prometheus-adapter/**"
          - "**/external-dns/**"
      host_metrics:
        filesystem:
          devices:
            excludes: [binfmt_misc]
          filesystems:
            excludes: [binfmt_misc]
          mountPoints:
            excludes: ["*/proc/sys/fs/binfmt_misc"]
        type: host_metrics
      internal_metrics:
        type: internal_metrics
    sinks:
      logtail:
        type: http
        inputs: [kubernetes_logs]
        uri: https://in.logtail.com/
        encoding:
          codec: json
        auth:
          strategy: bearer
          token: xxxxxxxxxx
      prom_exporter:
        type: prometheus_exporter
        inputs: [host_metrics, internal_metrics]
        address: 0.0.0.0:9090

[jszwedko]

Hi @mindrunner !

Using vector.dev/exclude: true should work. We have that functionality under test:

vector/lib/k8s-e2e-tests/tests/vector-agent.rs

Lines 619 to 812 in eb1aae2

	#[tokio::test]
	async fn pod_filtering() -> Result<(), Box<dyn std::error::Error>> {
	let _guard = lock();
	init();
	let namespace = get_namespace();
	let pod_namespace = get_namespace_appended(&namespace, "test-pod");
	let affinity_label = format!("{}-affinity", pod_namespace);
	let framework = make_framework();
	let override_name = get_override_name(&namespace, "vector-agent");
	let vector = framework
	.helm_chart(
	&namespace,
	"vector",
	"vector",
	"https://helm.vector.dev",
	VectorConfig {
	custom_helm_values: vec![
	&config_override_name(&override_name, true),
	HELM_VALUES_AGENT,
	],
	..Default::default()
	},
	)
	.await?;
	framework
	.wait_for_rollout(
	&namespace,
	&format!("daemonset/{}", override_name),
	vec!["--timeout=60s"],
	)
	.await?;
	let test_namespace = framework
	.namespace(namespace::Config::from_namespace(
	&namespace::make_namespace(pod_namespace.clone(), None),
	)?)
	.await?;
	let affinity_ns_name = format!("{}-affinity", pod_namespace);
	let affinity_ns = framework
	.namespace(namespace::Config::from_namespace(
	&namespace::make_namespace(affinity_ns_name.clone(), None),
	)?)
	.await?;
	let affinity_pod = create_affinity_pod(&framework, &affinity_ns_name, &affinity_label).await?;
	let excluded_test_pod = framework
	.test_pod(test_pod::Config::from_pod(&make_test_pod_with_affinity(
	&pod_namespace,
	"test-pod-excluded",
	"echo EXCLUDED_MARKER",
	vec![("vector.dev/exclude", "true")],
	vec![],
	Some((&affinity_label, "yes")),
	Some(&affinity_ns_name),
	))?)
	.await?;
	framework
	.wait(
	&pod_namespace,
	vec!["pods/test-pod-excluded"],
	WaitFor::Condition("initialized"),
	vec!["--timeout=60s"],
	)
	.await?;
	// Create this pod with affinity to the previous one to ensure they are deployed on the same
	// node.
	let control_test_pod = framework
	.test_pod(test_pod::Config::from_pod(&make_test_pod_with_affinity(
	&pod_namespace,
	"test-pod-control",
	"echo CONTROL_MARKER",
	vec![],
	vec![],
	Some((&affinity_label, "yes")),
	Some(&affinity_ns_name),
	))?)
	.await?;
	framework
	.wait(
	&pod_namespace,
	vec!["pods/test-pod-control"],
	WaitFor::Condition("initialized"),
	vec!["--timeout=60s"],
	)
	.await?;
	// Make sure we read the correct nodes logs.
	let vector_pod = framework
	.get_vector_pod_with_pod(
	&pod_namespace,
	"test-pod-control",
	&namespace,
	&override_name,
	)
	.await?;
	let mut log_reader = framework.logs(&namespace, &format!("pod/{}", vector_pod))?;
	smoke_check_first_line(&mut log_reader).await;
	// Read the log lines until the reasonable amount of time passes for us
	// to be confident that vector should've picked up the excluded message
	// if it wasn't filtering it.
	let mut got_control_marker = false;
	let mut lines_till_we_give_up: usize = 10000;
	let (stop_tx, mut stop_rx) = futures::channel::mpsc::channel(0);
	loop {
	let line = tokio::select! {
	result = stop_rx.next() => {
	result.unwrap();
	log_reader.kill().await?;
	continue;
	}
	line = log_reader.read_line() => line,
	};
	let line = match line {
	Some(line) => line,
	None => break,
	};
	debug!("Got line: {:?}", line);
	lines_till_we_give_up -= 1;
	if lines_till_we_give_up == 0 {
	info!("Giving up");
	log_reader.kill().await?;
	break;
	}
	if !line.starts_with('{') {
	// This isn't a json, must be an entry from Vector's own log stream.
	continue;
	}
	let val = parse_json(&line)?;
	if val["kubernetes"]["pod_namespace"] != pod_namespace.as_str() {
	// A log from something other than our test pod, pretend we don't
	// see it.
	continue;
	}
	// Ensure we got the log event from the control pod.
	assert_eq!(val["kubernetes"]["pod_name"], "test-pod-control");
	// Ensure the test sanity by validating that we got the control marker.
	// If we get an excluded marker here - it's an error.
	assert_eq!(val["message"], "CONTROL_MARKER");
	if got_control_marker {
	// We've already seen one control marker! This is not good, we only
	// emitted one.
	panic!("Control marker seen more than once");
	}
	// Remember that we've seen a control marker.
	got_control_marker = true;
	// Request termination in a while.
	let mut stop_tx = stop_tx.clone();
	tokio::spawn(async move {
	// Wait for two minutes - a reasonable time for vector internals to
	// pick up new `Pod` and collect events from them in idle load.
	// Here, we're assuming that if the `Pod` that was supposed to be
	// ignored was in fact collected (meaning something's wrong with
	// the exclusion logic), we'd see it's data within this time frame.
	// It's not enough to just wait for `Pod` complete, we should still
	// apply a reasonably big timeout before we stop waiting for the
	// logs to appear to have high confidence that Vector has enough
	// time to pick them up and spit them out.
	let duration = std::time::Duration::from_secs(120);
	info!("Starting stop timer, due in {} seconds", duration.as_secs());
	tokio::time::sleep(duration).await;
	info!("Stop timer complete");
	stop_tx.send(()).await.unwrap();
	});
	}
	// Ensure log reader exited.
	log_reader.wait().await.expect("log reader wait failed");
	assert!(got_control_marker);
	drop(excluded_test_pod);
	drop(control_test_pod);
	drop(affinity_pod);
	drop(affinity_ns);
	drop(test_namespace);
	drop(vector);
	Ok(())
	}

I'm not sure why it isn't working for you though 🤔 When you query for pods with vector.dev/exclude!=true do you see the pods with that label returned?

[mindrunner]

exclude_paths_glob_patterns seems to work now. It seems that there is a weird delay after activating this. I am not sure if that's related to vector or logtail tho.