Skip to content

Commit 6c3f42a

Browse files
authored
Merge master to logging-audit branch and bump up version 0.8.0-rc.6 (#35)
(1) merge master branch to logging-audit branch (2) init HOST_NAME to be "n/a" in LogbackThriftLogger (3) bump up version 0.8.0-rc.6
1 parent ed4c023 commit 6c3f42a

File tree

24 files changed

+256
-180
lines changed

24 files changed

+256
-180
lines changed

.github/workflows/maven.yml

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
name: Singer Build
2+
3+
on: [pull_request]
4+
5+
jobs:
6+
build:
7+
8+
runs-on: ubuntu-latest
9+
10+
steps:
11+
- uses: actions/checkout@v1
12+
- name: Set up JDK 1.8
13+
uses: actions/setup-java@v1
14+
with:
15+
java-version: 1.8
16+
- name: Build with Maven
17+
run: mvn -B package --file pom.xml

CONTRIBUTING.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,9 @@ To increase the chances that your pull request will be accepted:
2323
- Write tests for your changes
2424
- Write a good commit message
2525

26+
NOTE: all PRs must pass build before they can be accepted.
27+
2628
## License
2729

2830
By contributing to this project, you agree that your contributions will be
29-
licensed under its [Apache 2 license](LICENSE).
31+
licensed under its [Apache 2 license](LICENSE).

pom.xml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
<modelVersion>4.0.0</modelVersion>
77
<groupId>com.pinterest.singer</groupId>
88
<artifactId>singer-package</artifactId>
9-
<version>0.8.0-rc.5</version>
9+
<version>0.8.0-rc.6</version>
1010
<packaging>pom</packaging>
1111
<description>Singer Logging Agent modules</description>
1212
<inceptionYear>2013</inceptionYear>
@@ -37,6 +37,10 @@
3737
<id>ambud</id>
3838
<name>Ambud Sharma</name>
3939
</developer>
40+
<developer>
41+
<id>zzhhhzz</id>
42+
<name>Heng Zhang</name>
43+
</developer>
4044
</developers>
4145
<scm>
4246
<connection>https://github.com/pinterest/singer.git</connection>
@@ -51,10 +55,6 @@
5155
</modules>
5256

5357
<repositories>
54-
<repository>
55-
<id>Twitter public Maven repo</id>
56-
<url>https://maven.twttr.com</url>
57-
</repository>
5858
<repository>
5959
<id>central</id>
6060
<url>https://repo.maven.apache.org/maven2</url>

singer-commons/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
<parent>
2121
<groupId>com.pinterest.singer</groupId>
2222
<artifactId>singer-package</artifactId>
23-
<version>0.8.0-rc.5</version>
23+
<version>0.8.0-rc.6</version>
2424
<relativePath>../pom.xml</relativePath>
2525
</parent>
2626
<developers>

singer/deb.version

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
0.8.0-rc.5
1+
0.8.0-rc.6

singer/pom.xml

Lines changed: 5 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
<parent>
88
<groupId>com.pinterest.singer</groupId>
99
<artifactId>singer-package</artifactId>
10-
<version>0.8.0-rc.5</version>
10+
<version>0.8.0-rc.6</version>
1111
<relativePath>../pom.xml</relativePath>
1212
</parent>
1313
<licenses>
@@ -27,6 +27,10 @@
2727
<id>ambud</id>
2828
<name>Ambud Sharma</name>
2929
</developer>
30+
<developer>
31+
<id>zzhhhzz</id>
32+
<name>Heng Zhang</name>
33+
</developer>
3034
</developers>
3135
<scm>
3236
<connection>https://github.com/pinterest/singer.git</connection>
@@ -63,38 +67,6 @@
6367
<version>0.0.116</version>
6468
</dependency>
6569
<!-- https://mvnrepository.com/artifact/com.twitter.common/dynamic-host-set -->
66-
<dependency>
67-
<groupId>com.twitter.common</groupId>
68-
<artifactId>dynamic-host-set</artifactId>
69-
<version>0.0.56</version>
70-
</dependency>
71-
<!-- https://mvnrepository.com/artifact/com.twitter.common/service-thrift -->
72-
<dependency>
73-
<groupId>com.twitter.common</groupId>
74-
<artifactId>service-thrift</artifactId>
75-
<version>1.0.55</version>
76-
<exclusions>
77-
<exclusion>
78-
<groupId>org.apache.thrift</groupId>
79-
<artifactId>libthrift</artifactId>
80-
</exclusion>
81-
</exclusions>
82-
</dependency>
83-
<dependency>
84-
<groupId>com.twitter</groupId>
85-
<artifactId>finagle-core_2.11</artifactId>
86-
<version>6.25.0</version>
87-
</dependency>
88-
<dependency>
89-
<groupId>com.twitter</groupId>
90-
<artifactId>finagle-serversets_2.11</artifactId>
91-
<version>6.25.0</version>
92-
</dependency>
93-
<dependency>
94-
<groupId>com.twitter</groupId>
95-
<artifactId>util-core_2.11</artifactId>
96-
<version>6.25.0</version>
97-
</dependency>
9870
<dependency>
9971
<groupId>org.apache.kafka</groupId>
10072
<artifactId>kafka_2.11</artifactId>

singer/src/main/java/com/pinterest/singer/common/SingerLog.java

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,17 @@ public class SingerLog {
2727

2828
// The config for the SingerLog.
2929
private final SingerLogConfig singerLogConfig;
30+
private String podUid;
3031

3132
public SingerLog(SingerLogConfig singerLogConfig) {
3233
this.singerLogConfig = Preconditions.checkNotNull(singerLogConfig);
3334
}
3435

36+
public SingerLog(SingerLogConfig singerLogConfig, String podUid) {
37+
this.podUid = podUid;
38+
this.singerLogConfig = Preconditions.checkNotNull(singerLogConfig);
39+
}
40+
3541
@Override
3642
public boolean equals(Object o) {
3743
if (this == o) {
@@ -62,4 +68,8 @@ public String getLogName() {
6268
public SingerLogConfig getSingerLogConfig() {
6369
return singerLogConfig;
6470
}
71+
72+
public String getPodUid() {
73+
return podUid;
74+
}
6575
}

singer/src/main/java/com/pinterest/singer/common/SingerMetrics.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@ public class SingerMetrics {
7575
public static final String KAFKA_LATENCY = SINGER_WRITER + "max_kafka_batch_write_latency";
7676
public static final String NUM_COMMITED_TRANSACTIONS = SINGER_WRITER + "num_committed_transactions";
7777
public static final String NUM_ABORTED_TRANSACTIONS = SINGER_WRITER + "num_aborted_transactions";
78+
public static final String NUM_KAFKA_PRODUCERS = SINGER_WRITER + "num_kafka_producers";
7879

7980
public static final String KUBE_PREFIX = SINGER_PREIX + "kube.";
8081
public static final String KUBE_API_ERROR = KUBE_PREFIX + "api_error";

singer/src/main/java/com/pinterest/singer/config/ConfigFileServerSet.java

Lines changed: 10 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -18,15 +18,8 @@
1818
import com.google.common.annotations.VisibleForTesting;
1919
import com.google.common.base.Preconditions;
2020
import com.google.common.collect.ImmutableSet;
21-
import com.twitter.common.base.Command;
2221
import com.twitter.common.base.MorePreconditions;
23-
import com.twitter.common.zookeeper.Group;
24-
import com.twitter.common.zookeeper.ServerSet;
25-
import com.twitter.thrift.Endpoint;
26-
import com.twitter.thrift.ServiceInstance;
27-
import com.twitter.thrift.Status;
2822
import com.twitter.util.ExceptionalFunction;
29-
import org.apache.commons.lang.StringUtils;
3023

3124
import java.io.BufferedReader;
3225
import java.io.ByteArrayInputStream;
@@ -35,8 +28,6 @@
3528
import java.io.InputStream;
3629
import java.io.InputStreamReader;
3730
import java.net.InetSocketAddress;
38-
import java.util.Collections;
39-
import java.util.Map;
4031

4132
/**
4233
* Implementation of the ServerSet interface that uses a file on local disk instead of talking to ZooKeeper directly.
@@ -47,7 +38,7 @@
4738
* Note that this implementation only supports monitor() and not join(). Use the standard ZooKeeper implementation for
4839
* join().
4940
*/
50-
public class ConfigFileServerSet implements ServerSet {
41+
public class ConfigFileServerSet {
5142

5243
// made public for unit testing only
5344
public static String SERVERSET_DIR = "/var/serverset";
@@ -83,35 +74,7 @@ public ConfigFileServerSet(String serverSetFilePath) {
8374
}
8475
}
8576

86-
@Override
87-
public EndpointStatus join(
88-
InetSocketAddress endpoint, Map<String, InetSocketAddress> additionalEndpoints, Status status)
89-
throws Group.JoinException, InterruptedException {
90-
throw new UnsupportedOperationException("ConfigFileServerSet does not support join()");
91-
}
92-
93-
@Override
94-
public EndpointStatus join(
95-
InetSocketAddress endpoint, Map<String, InetSocketAddress> additionalEndpoints)
96-
throws Group.JoinException, InterruptedException {
97-
throw new UnsupportedOperationException("ConfigFileServerSet does not support join()");
98-
}
99-
100-
@Override
101-
public EndpointStatus join(
102-
InetSocketAddress endpoint, Map<String, InetSocketAddress> additionalEndpoints, int shardId)
103-
throws Group.JoinException, InterruptedException {
104-
throw new UnsupportedOperationException("ConfigFileServerSet does not support join()");
105-
}
106-
107-
@Override
108-
public Command watch(final HostChangeMonitor<ServiceInstance> monitor) throws MonitorException {
109-
monitor(monitor);
110-
return null;
111-
}
112-
113-
@Override
114-
public void monitor(final HostChangeMonitor<ServiceInstance> monitor) throws MonitorException {
77+
public void monitor(final ServersetMonitor monitor) throws Exception {
11578
Preconditions.checkNotNull(monitor);
11679
try {
11780
// Each call to monitor registers a new file watch. This is a bit inefficient if there
@@ -122,30 +85,30 @@ public void monitor(final HostChangeMonitor<ServiceInstance> monitor) throws Mon
12285
new ExceptionalFunction<byte[], Void>() {
12386
@Override
12487
public Void applyE(byte[] newContents) throws Exception {
125-
ImmutableSet<ServiceInstance> newServerSet = readServerSet(newContents);
88+
ImmutableSet<InetSocketAddress> newServerSet = readServerSet(newContents);
12689
monitor.onChange(newServerSet);
12790
return null;
12891
}
12992
});
13093
} catch (IOException e) {
131-
throw new MonitorException(
94+
throw new Exception(
13295
"Error setting up watch on dynamic server set file:" + serverSetFilePath, e);
13396
}
13497
}
13598

136-
protected Endpoint getEndPointFromServerSetLine(String line) {
99+
protected InetSocketAddress getEndPointFromServerSetLine(String line) {
137100
// We expect each line to be of the form "hostname:port". Note that host names can
138101
// contain ':' themselves (e.g. ipv6 addresses).
139102
int index = line.lastIndexOf(':');
140103
Preconditions.checkArgument(index > 0 && index < line.length() - 1);
141104

142105
String host = line.substring(0, index);
143106
int port = Integer.parseInt(line.substring(index + 1));
144-
return new Endpoint(host, port);
107+
return new InetSocketAddress(host, port);
145108
}
146109

147-
public ImmutableSet<ServiceInstance> readServerSet(byte[] fileContent) throws IOException {
148-
ImmutableSet.Builder<ServiceInstance> builder = new ImmutableSet.Builder<>();
110+
public ImmutableSet<InetSocketAddress> readServerSet(byte[] fileContent) throws IOException {
111+
ImmutableSet.Builder<InetSocketAddress> builder = new ImmutableSet.Builder<>();
149112
InputStream stream = new ByteArrayInputStream(fileContent);
150113
BufferedReader reader = new BufferedReader(new InputStreamReader(stream));
151114
while (true) {
@@ -158,12 +121,9 @@ public ImmutableSet<ServiceInstance> readServerSet(byte[] fileContent) throws IO
158121
continue;
159122
}
160123

161-
Endpoint endpoint = getEndPointFromServerSetLine(line);
124+
InetSocketAddress endpoint = getEndPointFromServerSetLine(line);
162125
if (endpoint != null) {
163-
builder.add(new ServiceInstance(
164-
endpoint, // endpoint
165-
Collections.<String, Endpoint>emptyMap(), // additional endpoints
166-
Status.ALIVE)); // status
126+
builder.add(endpoint);
167127
}
168128
}
169129
return builder.build();
Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package com.pinterest.singer.config;
2+
3+
import java.net.InetSocketAddress;
4+
5+
import com.google.common.collect.ImmutableSet;
6+
7+
public interface ServersetMonitor {
8+
9+
public void onChange(ImmutableSet<InetSocketAddress> serviceInstances);
10+
11+
}

0 commit comments

Comments
 (0)