fixed JedisDataException on json null values and added json unit test

Author: jruaux

README.adoc

@@ -20,7 +20,11 @@ Follow the usage help provided with the `--help` option.
 
 == Examples
 
-=== Beers
+=== Importing Files
+
+==== CSV
+
+===== Beers
 
 Here is an excerpt of https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv[beers.csv] available at https://github.com/nickhould/craft-beers-dataset[craft-beers-dataset]:  
 [source]
@@ -32,29 +36,28 @@ Here is an excerpt of https://raw.githubusercontent.com/nickhould/craft-beers-da
 3,0.09,,2263,Sinister,American Double / Imperial IPA,177,12.0
 ----
 
-==== Import to Hashes
+* Import beers to Hashes
 [source,shell]
 ----
 riot import csv --url https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv --header hash --keyspace beers --keys id
 ----
 
-==== Import to RediSearch Index
-. Create an index with redis-cli
+* Import beers to RediSearch
++
+.Create an index with redis-cli
 +
 [source,shell]
 ----
 FT.CREATE beerIdx SCHEMA abv NUMERIC SORTABLE id TAG name TEXT PHONETIC dm:en style TEXT PHONETIC dm:en brewery_id TAG ounces NUMERIC SORTABLE
 ----
-. Import data into the index
+.Import data into the index
 +
 [source,shell]
 ----
 riot import csv --header --included-fields 1 3 4 5 6 --url https://raw.githubusercontent.com/nickhould/craft-beers-dataset/master/data/processed/beers.csv search --index beerIdx --keys id
 ----
 
-=== OpenFlights
-
-==== Airports
+===== OpenFlights
 
 Here is an excerpt of https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat[airports.dat] available at https://github.com/jpatokal/openflights[openflights]
 ----
@@ -63,28 +66,22 @@ Here is an excerpt of https://raw.githubusercontent.com/jpatokal/openflights/mas
 3,"Mount Hagen Kagamuga Airport","Mount Hagen","Papua New Guinea","HGU","AYMH",-5.826789855957031,144.29600524902344,5388,10,"U","Pacific/Port_Moresby","airport","OurAirports"
 ----
 
-===== Airport Geo
-. Import airports into a geoset
-+
+.Import airports into a geoset
 [source,shell]
 ----
 riot import csv --url https://raw.githubusercontent.com/jpatokal/openflights/master/data/airports.dat --fields AirportID Name City Country IATA ICAO Latitude Longitude Altitude Timezone DST Tz Type Source geo --keyspace airportgeo --fields AirportID --longitude-field Longitude --latitude-field Latitude
 ----
-. Query the `airportgeo` geoset using redis-cli
-+
+.Query the `airportgeo` geoset using redis-cli
 [source,shell]
 ----
-> GEORADIUS airportgeo -122.4194 37.7749 20 mi
+GEORADIUS airportgeo -122.4194 37.7749 20 mi
 1) "3469"
 2) "10360"
 3) "8982"
 4) "9071"
 5) "3453"
-----
-+
-[source,shell]
-----
-> HGETALL airport:3469
+
+HGETALL airport:3469
  1) "AirportID"
  2) "3469"
  3) "ICAO"
@@ -115,5 +112,51 @@ riot import csv --url https://raw.githubusercontent.com/jpatokal/openflights/mas
 28) "13"
 ----
 
-
+==== JSON
+
+===== Beers
+
+Here is an excerpt of beers.json available at https://github.com/rethinkdb/beerthink/blob/master/data/beers.json[beerthink]:
+[source,json]
+----
+[
+	{
+		"id": "1",
+		"brewery_id": "812",
+		"name": "Hocus Pocus",
+		"abv": "4.5",
+		"ibu": "0",
+		"srm": "0",
+		"upc": "0",
+		"filepath": "",
+		"descript": "Our take on a classic summer ale.  A toast to weeds, rays, and summer haze.  A light, crisp ale for mowing lawns, hitting lazy fly balls, and communing with nature, Hocus Pocus is offered up as a summer sacrifice to clodless days.\n\nIts malty sweetness finishes tart and crisp and is best apprediated with a wedge of orange.",
+		"add_user": "0",
+		"last_mod": "2010-07-22 20:00:20 UTC",
+		"style_name": "Light American Wheat Ale or Lager",
+		"cat_name": "Other Style"
+	},
+	{
+		"id": "6",
+		"brewery_id": "1385",
+		"name": "Winter Warmer",
+		"abv": "5.199999809265137",
+		"ibu": "0",
+		"srm": "0",
+		"upc": "0",
+		"filepath": "",
+		"descript": "",
+		"add_user": "0",
+		"last_mod": "2010-07-22 20:00:20 UTC",
+		"style_name": "Old Ale",
+		"cat_name": "British Ale"
+	},
+	...
+]
+----
+
+.Import beers into Hashes
+[source,shell]
+----
+riot import json --url=https://raw.githubusercontent.com/rethinkdb/beerthink/master/data/beers.json hash --keyspace beerjson --keys id
+----
 

pom.xml

@@ -12,7 +12,7 @@
 
 	<groupId>com.redislabs</groupId>
 	<artifactId>riot</artifactId>
-	<version>0.2.0</version>
+	<version>0.2.1</version>
 	<packaging>jar</packaging>
 	<name>RIOT</name>
 	<description>Redis Input/Output Tool</description>
@@ -58,7 +58,7 @@
 		<dependency>
 			<groupId>com.redislabs</groupId>
 			<artifactId>lettusearch</artifactId>
-			<version>1.1.9</version>
+			<version>1.2.0</version>
 		</dependency>
 		<dependency>
 			<groupId>org.apache.commons</groupId>

src/main/java/com/redislabs/riot/cli/AbstractCommand.java

@@ -106,7 +106,7 @@ public void run(String sourceDescription, AbstractItemCountingItemStreamItemRead
 		builder.setSleep(sleep);
 		Job job = builder.build();
 		long startTime = System.currentTimeMillis();
-		System.out.println("Importing from " + sourceDescription + " to " + targetDescription);
+		System.out.println("Importing " + sourceDescription + " into " + targetDescription);
 		SimpleJobLauncher jobLauncher = new SimpleJobLauncher();
 		jobLauncher.setJobRepository(builder.jobRepository());
 		jobLauncher.setTaskExecutor(new SyncTaskExecutor());

src/main/java/com/redislabs/riot/cli/redis/AbstractCollectionRedisImportSubSubCommand.java

@@ -6,9 +6,23 @@
 
 public abstract class AbstractCollectionRedisImportSubSubCommand extends AbstractRedisImportSubSubCommand {
 
+	@Option(names = "--keyspace", required = true, description = "Redis keyspace prefix.")
+	private String keyspace;
+	@Option(names = "--keys", arity = "1..*", description = "Key fields.")
+	private String[] keys = new String[0];
 	@Option(names = "--fields", required = true, arity = "1..*", description = "Fields used to build member ids for collection data structures (list, set, zset, geo).")
 	private String[] fields;
 
+	@Override
+	protected String getKeyspace() {
+		return keyspace;
+	}
+
+	@Override
+	protected String[] getKeys() {
+		return keys;
+	}
+
 	@Override
 	protected AbstractCollectionRedisItemWriter redisItemWriter() {
 		AbstractCollectionRedisItemWriter writer = collectionRedisItemWriter();

src/main/java/com/redislabs/riot/cli/redis/AbstractRediSearchImportSubSubCommand.java

@@ -13,9 +13,23 @@
 
 public abstract class AbstractRediSearchImportSubSubCommand extends AbstractRedisImportSubSubCommand {
 
+	@Option(names = { "--keyspace" }, description = "Document keyspace prefix.")
+	private String keyspace;
+	@Option(names = "--keys", required = true, arity = "1..*", description = "Document key fields.")
+	private String[] keys = new String[0];
 	@Getter
 	@Option(names = "--index", description = "Name of the RediSearch index", required = true)
 	private String index;
+	
+	@Override
+	protected String getKeyspace() {
+		return keyspace;
+	}
+	
+	@Override
+	protected String[] getKeys() {
+		return keys;
+	}
 
 	@Override
 	protected ItemStreamWriter<Map<String, Object>> writer() {

src/main/java/com/redislabs/riot/cli/redis/AbstractRedisImportSubSubCommand.java

@@ -4,15 +4,8 @@
 import com.redislabs.riot.redis.RedisConverter;
 import com.redislabs.riot.redis.writer.AbstractRedisItemWriter;
 
-import picocli.CommandLine.Option;
-
 public abstract class AbstractRedisImportSubSubCommand extends AbstractImportSubSubCommand {
 
-	@Option(names = { "-s", "--keyspace" }, description = "Redis keyspace prefix.")
-	private String keyspace;
-	@Option(names = { "-k", "--keys" }, arity = "1..*", description = "Key fields.")
-	private String[] keys = new String[0];
-
 	@Override
 	public String getTargetDescription() {
 		return getDataStructure() + " \"" + getKeyspaceDescription() + "\"";
@@ -21,8 +14,8 @@ public String getTargetDescription() {
 	protected abstract String getDataStructure();
 
 	protected String getKeyspaceDescription() {
-		String description = keyspace == null ? "" : keyspace;
-		for (String key : keys) {
+		String description = getKeyspace() == null ? "" : getKeyspace();
+		for (String key : getKeys()) {
 			description += ":<" + key + ">";
 		}
 		return description;
@@ -37,11 +30,15 @@ protected AbstractRedisItemWriter itemWriter() {
 
 	protected RedisConverter redisConverter() {
 		RedisConverter converter = new RedisConverter();
-		converter.setKeyspace(keyspace);
-		converter.setKeys(keys);
+		converter.setKeyspace(getKeyspace());
+		converter.setKeys(getKeys());
 		return converter;
 	}
 
 	protected abstract AbstractRedisItemWriter redisItemWriter();
 
+	protected abstract String getKeyspace();
+
+	protected abstract String[] getKeys();
+
 }

src/main/java/com/redislabs/riot/cli/redis/AbstractSingleRedisImportSubSubCommand.java

@@ -0,0 +1,22 @@
+package com.redislabs.riot.cli.redis;
+
+import picocli.CommandLine.Option;
+
+public abstract class AbstractSingleRedisImportSubSubCommand extends AbstractRedisImportSubSubCommand {
+
+	@Option(names = "--keyspace", required = true, description = "Redis keyspace prefix.")
+	private String keyspace;
+	@Option(names = "--keys", arity = "1..*", required = true, description = "Key fields.")
+	private String[] keys = new String[0];
+
+	@Override
+	protected String getKeyspace() {
+		return keyspace;
+	}
+
+	@Override
+	protected String[] getKeys() {
+		return keys;
+	}
+
+}

src/main/java/com/redislabs/riot/cli/redis/HashImportSubSubCommand.java

@@ -6,7 +6,7 @@
 import picocli.CommandLine.Command;
 
 @Command(name = "hash", description = "Hash data structure")
-public class HashImportSubSubCommand extends AbstractRedisImportSubSubCommand {
+public class HashImportSubSubCommand extends AbstractSingleRedisImportSubSubCommand {
 
 	@Override
 	protected AbstractRedisItemWriter redisItemWriter() {

src/main/java/com/redislabs/riot/cli/redis/StreamImportSubSubCommand.java

@@ -7,7 +7,7 @@
 import picocli.CommandLine.Option;
 
 @Command(name = "stream", description = "Stream data structure")
-public class StreamImportSubSubCommand extends AbstractRedisImportSubSubCommand {
+public class StreamImportSubSubCommand extends AbstractSingleRedisImportSubSubCommand {
 
 	@Option(names = "--approximate-trimming", description = "Apply efficient trimming for capped streams using the ~ flag.")
 	private boolean approximateTrimming;

src/main/java/com/redislabs/riot/cli/redis/StringImportSubSubCommand.java

@@ -8,7 +8,7 @@
 import picocli.CommandLine.Option;
 
 @Command(name = "string", description = "String data structure")
-public class StringImportSubSubCommand extends AbstractRedisImportSubSubCommand {
+public class StringImportSubSubCommand extends AbstractSingleRedisImportSubSubCommand {
 
 	@Option(names = "--format", description = "Serialization format: ${COMPLETION-CANDIDATES}")
 	private StringFormat format = StringFormat.Json;

src/main/java/com/redislabs/riot/redis/writer/AbstractRedisCommands.java

@@ -12,10 +12,18 @@ public abstract class AbstractRedisCommands implements RedisCommands {
 
 	protected Map<String, String> stringMap(Map<String, Object> item) {
 		Map<String, String> stringMap = new HashMap<String, String>();
-		for (String key : item.keySet()) {
-			Object value = item.get(key);
-			stringMap.put(key, converter.convert(value, String.class));
-		}
+		item.forEach((k, v) -> put(stringMap, k, v));
 		return stringMap;
 	}
+
+	private void put(Map<String, String> map, String key, Object value) {
+		if (value == null) {
+			return;
+		}
+		if (value instanceof Map) {
+			((Map<?, ?>) value).forEach((k, v) -> put(map, key + "." + converter.convert(k, String.class), v));
+		} else {
+			map.put(key, converter.convert(value, String.class));
+		}
+	}
 }