Skip to content

Commit 49a2362

Browse files
simonpoolesimonpoole
committed
Add replication file generation and replication from files
This adds a facility to generate replication files in the "classical" OSM directory layout style and a facility to consume such files from a remote location via http.
1 parent b97714e commit 49a2362

File tree

10 files changed

+776
-15
lines changed

10 files changed

+776
-15
lines changed

‎pom.xml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,11 @@
156156
<version>1.9.2</version>
157157
<scope>test</scope>
158158
</dependency>
159-
159+
<dependency>
160+
<groupId>com.cedarsoftware</groupId>
161+
<artifactId>json-io</artifactId>
162+
<version>4.10.1</version>
163+
</dependency>
160164
</dependencies>
161165

162166
<repositories>

‎src/main/java/de/komoot/photon/App.java‎

Lines changed: 39 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
package de.komoot.photon;
22

3-
43
import com.beust.jcommander.JCommander;
54
import com.beust.jcommander.ParameterException;
5+
6+
import de.komoot.photon.elasticsearch.ReplicationUpdater;
7+
import de.komoot.photon.elasticsearch.Replicatior;
68
import de.komoot.photon.elasticsearch.Server;
79
import de.komoot.photon.nominatim.NominatimConnector;
810
import de.komoot.photon.nominatim.NominatimUpdater;
@@ -12,12 +14,12 @@
1214
import spark.Request;
1315
import spark.Response;
1416

17+
import java.io.File;
1518
import java.io.FileNotFoundException;
1619
import java.io.IOException;
1720

1821
import static spark.Spark.*;
1922

20-
2123
@Slf4j
2224
public class App {
2325

@@ -47,6 +49,11 @@ public static void main(String[] rawArgs) throws Exception {
4749
return;
4850
}
4951

52+
if (args.isReplicate()) {
53+
startReplicationUpdate(args);
54+
return;
55+
}
56+
5057
boolean shutdownES = false;
5158
final Server esServer = new Server(args).start();
5259
try {
@@ -67,11 +74,11 @@ public static void main(String[] rawArgs) throws Exception {
6774
// no special action specified -> normal mode: start search API
6875
startApi(args, esClient);
6976
} finally {
70-
if (shutdownES) esServer.shutdown();
77+
if (shutdownES)
78+
esServer.shutdown();
7179
}
7280
}
7381

74-
7582
/**
7683
* dump elastic search index and create a new and empty one
7784
*
@@ -87,7 +94,6 @@ private static void startRecreatingIndex(Server esServer) {
8794
log.info("deleted photon index and created an empty new one.");
8895
}
8996

90-
9197
/**
9298
* take nominatim data and dump it to json
9399
*
@@ -97,7 +103,8 @@ private static void startJsonDump(CommandLineArgs args) {
97103
try {
98104
final String filename = args.getJsonDump();
99105
final JsonDumper jsonDumper = new JsonDumper(filename, args.getLanguages());
100-
NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
106+
NominatimConnector nominatimConnector = new NominatimConnector(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(),
107+
args.getPassword());
101108
nominatimConnector.setImporter(jsonDumper);
102109
nominatimConnector.readEntireDatabase(args.getCountryCodes().split(","));
103110
log.info("json dump was created: " + filename);
@@ -106,7 +113,6 @@ private static void startJsonDump(CommandLineArgs args) {
106113
}
107114
}
108115

109-
110116
/**
111117
* take nominatim data to fill elastic search index
112118
*
@@ -130,11 +136,22 @@ private static void startNominatimImport(CommandLineArgs args, Server esServer,
130136
log.info("imported data from nominatim to photon with languages: " + args.getLanguages());
131137
}
132138

139+
/**
140+
* Run generation of replication files once
141+
*
142+
* @param args the arguments from the command line
143+
*/
144+
private static void startReplicationUpdate(CommandLineArgs args) {
145+
final NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
146+
Updater updater = new ReplicationUpdater(new File(args.getReplicationDirectory()), null);
147+
nominatimUpdater.setUpdater(updater);
148+
nominatimUpdater.update();
149+
}
133150

134151
/**
135152
* start api to accept search requests via http
136153
*
137-
* @param args
154+
* @param args the arguments from the command line
138155
* @param esNodeClient
139156
*/
140157
private static void startApi(CommandLineArgs args, Client esNodeClient) {
@@ -159,11 +176,19 @@ private static void startApi(CommandLineArgs args, Client esNodeClient) {
159176
// setup update API
160177
final NominatimUpdater nominatimUpdater = new NominatimUpdater(args.getHost(), args.getPort(), args.getDatabase(), args.getUser(), args.getPassword());
161178
Updater updater = new de.komoot.photon.elasticsearch.Updater(esNodeClient, args.getLanguages());
162-
nominatimUpdater.setUpdater(updater);
163-
164-
get("/nominatim-update", (Request request, Response response) -> {
165-
new Thread(() -> nominatimUpdater.update()).start();
166-
return "nominatim update started (more information in console output) ...";
167-
});
179+
String replicationUrl = args.getReplicationUrl();
180+
if (replicationUrl == null) {
181+
if (args.isGenerateFiles()) {
182+
updater = new ReplicationUpdater(new File(args.getReplicationDirectory()), updater);
183+
}
184+
nominatimUpdater.setUpdater(updater);
185+
get("/nominatim-update", (Request request, Response response) -> {
186+
new Thread(() -> nominatimUpdater.update()).start();
187+
return "nominatim update started (more information in console output) ...";
188+
});
189+
} else {
190+
Replicatior replicator = new Replicatior(new File(args.getReplicationDirectory()), args.getReplicationUrl(), args.getReplicationInterval(), updater);
191+
replicator.start();
192+
}
168193
}
169194
}

‎src/main/java/de/komoot/photon/CommandLineArgs.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,21 @@ public class CommandLineArgs {
6161

6262
@Parameter(names = "-cors-origin", description = "enable cross-site resource sharing for the specified origin (default CORS not supported)")
6363
private String corsOrigin = null;
64+
65+
@Parameter(names = "-replicate", description = "generate a replication file")
66+
private boolean replicate = false;
67+
68+
@Parameter(names = "-replication-dir", description = "directory for replication files (default '.')")
69+
private String replicationDirectory = new File(".").getAbsolutePath();
70+
71+
@Parameter(names = "-generate-files", description = "generate replication files while updating from Nominatim")
72+
private boolean generateFiles = false;
73+
74+
@Parameter(names = "-replication-url", description = "url of base directory for file based replication (default none)")
75+
private String replicationUrl = null;
76+
77+
@Parameter(names = "-replication-interval", description = "interval between trying to read replication files in minutes (default 60 minutes)")
78+
private int replicationInterval = 60;
6479

6580
@Parameter(names = "-h", description = "show help / usage")
6681
private boolean usage = false;

‎src/main/java/de/komoot/photon/elasticsearch/PhotonAction.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package de.komoot.photon.elasticsearch;
2+
3+
import de.komoot.photon.PhotonDoc;
4+
5+
/**
6+
* Container class for an update action and a PhotonDoc
7+
*
8+
* @author Simon
9+
*
10+
*/
11+
public class PhotonAction {
12+
13+
public enum ACTION {CREATE, DELETE, UPDATE, UPDATE_OR_CREATE};
14+
15+
public ACTION action;
16+
17+
public long id;
18+
19+
public PhotonDoc doc;
20+
}

‎src/main/java/de/komoot/photon/elasticsearch/ReplicationState.java‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
package de.komoot.photon.elasticsearch;
2+
3+
import java.io.BufferedInputStream;
4+
import java.io.ByteArrayOutputStream;
5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.net.HttpURLConnection;
8+
import java.net.URL;
9+
import java.nio.charset.Charset;
10+
import java.nio.file.Files;
11+
import java.nio.file.Path;
12+
import java.util.Date;
13+
14+
import javax.annotation.Nonnull;
15+
16+
import org.json.JSONObject;
17+
18+
import de.komoot.photon.utils.DateFormatter;
19+
import lombok.Getter;
20+
import lombok.Setter;
21+
22+
/**
23+
* Container for the replication state
24+
*
25+
* @author Simon
26+
*
27+
*/
28+
@Getter
29+
@Setter
30+
public class ReplicationState {
31+
32+
private static final String TIMESTAMP_KEY = "timestamp";
33+
private static final String FORMAT_KEY = "format";
34+
private static final String SEQUENCE_NUMBER_KEY = "sequenceNumber";
35+
36+
final long sequenceNumber;
37+
final String format;
38+
final Date timeStamp;
39+
40+
final DateFormatter dateFormatter;
41+
42+
/**
43+
* Construct a new instance
44+
*
45+
* @param sequenceNumber the current sequence number
46+
* @param format the format of the replication file
47+
* @param timeStamp the current time
48+
*/
49+
public ReplicationState(long sequenceNumber, @Nonnull String format, @Nonnull Date timeStamp) {
50+
this.sequenceNumber = sequenceNumber;
51+
this.format = format;
52+
this.timeStamp = timeStamp;
53+
dateFormatter = new DateFormatter();
54+
}
55+
56+
/**
57+
* Read the replication state from a local file
58+
*
59+
* @param stateFilePath the Path to the file
60+
* @return a ReplicationState instance
61+
* @throws IOException if something goes wrong
62+
*/
63+
@Nonnull
64+
public static ReplicationState readState(@Nonnull Path stateFilePath) throws IOException {
65+
String content = new String(Files.readAllBytes(stateFilePath), Charset.forName("UTF-8"));
66+
JSONObject state = new JSONObject(content);
67+
68+
return new ReplicationState(state.getLong(SEQUENCE_NUMBER_KEY), state.getString(FORMAT_KEY), null);
69+
}
70+
71+
/**
72+
* Read the replication state from an url
73+
*
74+
* @param urlString the url from the file
75+
* @return a ReplicationState instance
76+
* @throws IOException if something goes wrong
77+
*/
78+
@Nonnull
79+
public static ReplicationState readState(@Nonnull String urlString) throws IOException {
80+
URL url = new URL(urlString);
81+
HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection();
82+
urlConnection.setRequestProperty("User-Agent", "Photon");
83+
urlConnection.setInstanceFollowRedirects(true);
84+
try {
85+
InputStream in = new BufferedInputStream(urlConnection.getInputStream());
86+
ByteArrayOutputStream result = new ByteArrayOutputStream();
87+
byte[] buffer = new byte[1024];
88+
int length;
89+
while ((length = in.read(buffer)) != -1) {
90+
result.write(buffer, 0, length);
91+
}
92+
93+
JSONObject state = new JSONObject(result.toString("UTF-8"));
94+
return new ReplicationState(state.getLong(SEQUENCE_NUMBER_KEY), state.getString(FORMAT_KEY), null);
95+
} finally {
96+
urlConnection.disconnect();
97+
}
98+
}
99+
100+
/**
101+
* Get a JSON representation of the replication state
102+
*
103+
* @return a JSONObject containg the state
104+
*/
105+
@Nonnull
106+
public JSONObject getJson() {
107+
final JSONObject state = new JSONObject();
108+
state.put(SEQUENCE_NUMBER_KEY, sequenceNumber);
109+
state.put(FORMAT_KEY, format);
110+
state.put(TIMESTAMP_KEY, dateFormatter.format(timeStamp));
111+
return state;
112+
}
113+
114+
/**
115+
* Get a JSON representation of the replication state as a String
116+
*
117+
* @return a String containing JSON
118+
*/
119+
@Nonnull
120+
public String toJsonString() {
121+
return getJson().toString(4);
122+
}
123+
}

0 commit comments

Comments
 (0)