SDAP-161 MUDROD embedded unit test

core/src/main/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngine.java

@@ -54,7 +54,8 @@ public WeblogDiscoveryEngine(Properties props, ESDriver es, SparkDriver spark) {
   /**
    * Get log file list from a directory
    *
-   * @param logDir path to directory containing logs either local or in HDFS.
+   * @param logDir
+   *          path to directory containing logs either local or in HDFS.
    * @return a list of log files
    */
   public List<String> getFileList(String logDir) {

core/src/main/java/org/apache/sdap/mudrod/main/MudrodEngine.java

@@ -391,12 +391,12 @@ public static void main(String[] args) {
       me.end();
     } catch (Exception e) {
       HelpFormatter formatter = new HelpFormatter();
-      formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. " + "User must also provide an ingest method.", true);
+      formatter.printHelp("MudrodEngine: 'dataDir' argument is mandatory. " + "User must also provide an ingest method.", options, true);
       LOG.error("Error whilst parsing command line.", e);
     }
   }
 
-  private static void loadPathConfig(MudrodEngine me, String dataDir) {
+  public static void loadPathConfig(MudrodEngine me, String dataDir) {
     me.props.put(MudrodConstants.ONTOLOGY_INPUT_PATH, dataDir + "SWEET_ocean/");
     me.props.put(MudrodConstants.ONTOLOGY_PATH, dataDir + "ocean_triples.csv");
     me.props.put(MudrodConstants.USER_HISTORY_PATH, dataDir + "userhistorymatrix.csv");
@@ -405,7 +405,6 @@ private static void loadPathConfig(MudrodEngine me, String dataDir) {
     me.props.put(MudrodConstants.CLICKSTREAM_SVD_PATH, dataDir + "clickstreamsvdmatrix_tmp.csv");
     me.props.put(MudrodConstants.METADATA_SVD_PATH, dataDir + "metadatasvdMatrix_tmp.csv");
     me.props.put(MudrodConstants.RAW_METADATA_PATH, dataDir + me.props.getProperty(MudrodConstants.RAW_METADATA_TYPE));
-
     me.props.put(MudrodConstants.METADATA_TERM_MATRIX_PATH, dataDir + "metadata_term_tfidf.csv");
     me.props.put(MudrodConstants.METADATA_WORD_MATRIX_PATH, dataDir + "metadata_word_tfidf.csv");
     me.props.put(MudrodConstants.METADATA_SESSION_MATRIX_PATH, dataDir + "metadata_session_coocurrence_matrix.csv");
@@ -428,6 +427,5 @@ public SparkDriver getSparkDriver() {
    */
   public void setSparkDriver(SparkDriver sparkDriver) {
     this.spark = sparkDriver;
-
   }
 }

core/src/main/java/org/apache/sdap/mudrod/weblog/pre/ClickStreamGenerator.java

@@ -71,5 +71,4 @@ public Object execute() {
   public Object execute(Object o) {
     return null;
   }
-
 }

core/src/main/java/org/apache/sdap/mudrod/weblog/pre/SessionStatistic.java

@@ -229,7 +229,7 @@ public int processSession(ESDriver es, String sessionId) throws IOException, Int
               String[] keywordList = keywords.split(",");
               for (String item : items) {
                 if (!Arrays.asList(keywordList).contains(item)) {
-                  keywords = keywords + item + ",";
+                  keywords = keywords + "," + item + ",";
                 }
               }
             }

core/src/main/java/org/apache/sdap/mudrod/weblog/structure/session/SessionTree.java

@@ -26,10 +26,13 @@
 
 import java.io.UnsupportedEncodingException;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.Set;
 import java.util.concurrent.ExecutionException;
 
 /**
@@ -205,7 +208,7 @@ public List<ClickStream> getClickStreamList(Properties props) {
       RequestUrl requestURL = new RequestUrl();
       String viewquery = "";
       try {
-        String infoStr = requestURL.getSearchInfo(viewnode.getRequest());
+        String infoStr = requestURL.getSearchInfo(viewnode.getReferer());
         viewquery = es.customAnalyzing(props.getProperty(MudrodConstants.ES_INDEX_NAME), infoStr);
       } catch (UnsupportedEncodingException | InterruptedException | ExecutionException e) {
         LOG.warn("Exception getting search info. Ignoring...", e);
@@ -222,6 +225,8 @@ public List<ClickStream> getClickStreamList(Properties props) {
 
       if (viewquery != null && !"".equals(viewquery)) {
         String[] queries = viewquery.trim().split(",");
+        Set<String> queryset = new HashSet<>();
+        queryset.addAll(Arrays.asList(queries));
         if (queries.length > 0) {
           for (String query : queries) {
             ClickStream data = new ClickStream(query, dataset, download);

core/src/test/java/org/apache/sdap/mudrod/discoveryengine/WeblogDiscoveryEngineTest.java

@@ -0,0 +1,126 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.discoveryengine;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.sdap.mudrod.driver.ESDriver;
+import org.apache.sdap.mudrod.driver.SparkDriver;
+import org.apache.sdap.mudrod.main.AbstractElasticsearchIntegrationTest;
+import org.apache.sdap.mudrod.main.MudrodConstants;
+import org.apache.sdap.mudrod.main.MudrodEngine;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class WeblogDiscoveryEngineTest extends AbstractElasticsearchIntegrationTest {
+
+  private static WeblogDiscoveryEngine weblogEngine = null;
+
+  @BeforeClass
+  public static void setUp() {
+    MudrodEngine mudrodEngine = new MudrodEngine();
+    Properties props = mudrodEngine.loadConfig();
+    ESDriver es = new ESDriver(props);
+    SparkDriver spark = new SparkDriver(props);
+    String dataDir = getTestDataPath();
+    System.out.println(dataDir);
+    props.setProperty(MudrodConstants.DATA_DIR, dataDir);
+    MudrodEngine.loadPathConfig(mudrodEngine, dataDir);
+    weblogEngine = new WeblogDiscoveryEngine(props, es, spark);
+  }
+
+  @AfterClass
+  public static void tearDown() {
+  }
+
+  private static String getTestDataPath() {
+    File resourcesDirectory = new File("src/test/resources/");
+    String resourcedir = "/Testing_Data_1_3dayLog+Meta+Onto/";
+    String dataDir = resourcesDirectory.getAbsolutePath() + resourcedir;
+    return dataDir;
+  }
+
+  @Test
+  public void testPreprocess() throws IOException {
+    weblogEngine.preprocess();
+    testPreprocessUserHistory();
+    testPreprocessClickStream();
+  }
+
+  private void testPreprocessUserHistory() throws IOException {
+    // compare user history data
+    String userHistorycsvFile = getTestDataPath() + "/userHistoryMatrix.csv";
+    HashMap<String, List<String>> map = extractPairFromCSV(userHistorycsvFile);
+    Assert.assertEquals("failed in history data result!", "195.219.98.7", String.join(",", map.get("sea surface topography")));
+  }
+
+  private void testPreprocessClickStream() throws IOException {
+    String clickStreamcsvFile = getTestDataPath() + "/clickStreamMatrix.csv";
+    System.out.println(clickStreamcsvFile);
+    HashMap<String, List<String>> map = extractPairFromCSV(clickStreamcsvFile);
+    System.out.println(map);
+    Assert.assertEquals("failed in click stream result!", "\"ostm_l2_ost_ogdr_gps\"", String.join(",", map.get("sea surface topography")));
+  }
+
+  private HashMap extractPairFromCSV(String csvfile){
+
+    BufferedReader br = null;
+    try {
+      br = new BufferedReader(new FileReader(csvfile));
+    } catch (FileNotFoundException e) {
+      e.printStackTrace();
+    }
+    String line = null;
+    HashMap<String, List<String>> map = new HashMap<>();
+    int i = 0;
+    List<String> header = new LinkedList<>();
+    try {
+      while ((line = br.readLine()) != null) {
+        if (i == 0) {
+          String str[] = line.split(",");
+          for (String s : str) {
+            header.add(s);
+          }
+        } else {
+          String str[] = line.split(",");
+          for (int j = 1; j < str.length; j++) {
+            int value = Integer.parseInt(str[j]);
+            if (value > 0) {
+              if (!map.containsKey(str[0])) {
+                map.put(str[0], new ArrayList<>());
+              }
+              map.get(str[0]).add(header.get(j));
+            }
+          }
+        }
+        i += 1;
+      }
+    } catch (IOException e) {
+      e.printStackTrace();
+    }
+
+    return map;
+  }
+}

core/src/test/java/org/apache/sdap/mudrod/driver/EmbeddedElasticsearchServer.java

@@ -0,0 +1,77 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.driver;
+
+import org.apache.commons.io.FileUtils;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeValidationException;
+import org.elasticsearch.transport.Netty3Plugin;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+
+/**
+ * embedded elasticsearch server.
+ */
+public class EmbeddedElasticsearchServer {
+
+  private static final String DEFAULT_DATA_DIRECTORY = "target/elasticsearch-data";
+
+  private Node node;
+  private final String dataDirectory;
+
+  public EmbeddedElasticsearchServer() {
+    this(DEFAULT_DATA_DIRECTORY);
+  }
+
+  public EmbeddedElasticsearchServer(String dataDirectory) {
+    this.dataDirectory = dataDirectory;
+
+    Settings.Builder settingsBuilder = Settings.builder();
+    settingsBuilder.put("http.type", "netty3");
+    settingsBuilder.put("transport.type", "netty3");
+    settingsBuilder.put("cluster.name", "MurdorES").put("http.enabled", "false").put("path.data", dataDirectory).put("path.home", "/");
+
+    Settings settings = settingsBuilder.build();
+    Collection plugins = Arrays.asList(Netty3Plugin.class);
+    node = null;
+    try {
+      node = new PluginConfigurableNode(settings, plugins).start();
+      System.out.println(node.toString());
+    } catch (NodeValidationException e) {
+      e.printStackTrace();
+    }
+
+    System.out.println(node.getNodeEnvironment().nodeId());
+  }
+
+  public Client getClient() {
+    return node.client();
+  }
+
+  public void shutdown() {
+  }
+
+  private void deleteDataDirectory() {
+    try {
+      FileUtils.deleteDirectory(new File(dataDirectory));
+    } catch (IOException e) {
+      throw new RuntimeException("Could not delete data directory of embedded elasticsearch server", e);
+    }
+  }
+}

core/src/test/java/org/apache/sdap/mudrod/driver/PluginConfigurableNode.java

@@ -0,0 +1,27 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.driver;
+
+import java.util.Collection;
+
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.internal.InternalSettingsPreparer;
+import org.elasticsearch.plugins.Plugin;
+
+public class PluginConfigurableNode extends Node {
+  public PluginConfigurableNode(Settings settings, Collection<Class<? extends Plugin>> classpathPlugins) {
+    super(InternalSettingsPreparer.prepareEnvironment(settings, null), classpathPlugins);
+  }
+}

core/src/test/java/org/apache/sdap/mudrod/main/AbstractElasticsearchIntegrationTest.java

@@ -0,0 +1,48 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License"); you 
+ * may not use this file except in compliance with the License. 
+ * You may obtain a copy of the License at
+ * 
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sdap.mudrod.main;
+
+import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer;
+import org.apache.sdap.mudrod.driver.EmbeddedElasticsearchServer;
+import org.elasticsearch.client.Client;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+
+/**
+ * This is a helper class the starts an embedded elasticsearch server for each
+ * test.
+ */
+public abstract class AbstractElasticsearchIntegrationTest {
+
+  private static EmbeddedElasticsearchServer embeddedElasticsearchServer;
+
+  @BeforeClass
+  public static void startEmbeddedElasticsearchServer() {
+    embeddedElasticsearchServer = new EmbeddedElasticsearchServer();
+  }
+
+  @AfterClass
+  public static void shutdownEmbeddedElasticsearchServer() {
+    embeddedElasticsearchServer.shutdown();
+  }
+
+  /**
+   * By using this method you can access the embedded server.
+   */
+  protected Client getClient() {
+    return embeddedElasticsearchServer.getClient();
+  }
+}