Bugfix estimation of file extension.

Author: VolkerHartmann

src/main/java/edu/kit/datamanager/mappingservice/util/FileUtil.java

@@ -52,7 +52,7 @@ public class FileUtil {
   /**
    * Default value for suffix of temporary files.
    */
-  public static final String DEFAULT_SUFFIX = ".tmp";
+  public static final String DEFAULT_SUFFIX = ".html";
   /**
    * Default value for prefix of temporary files.
    */
@@ -61,9 +61,9 @@ public class FileUtil {
    * Logger for this class.
    */
   private static final Logger LOGGER = LoggerFactory.getLogger(FileUtil.class);
-
+  
   private static final int MAX_LENGTH_OF_HEADER = 100;
-
+  
   private static final Pattern JSON_FIRST_BYTE = Pattern.compile("(\\R\\s)*\\s*\\{\\s*\"(.|\\s)*", Pattern.MULTILINE);//^\\s{\\s*\".*");
   private static final Pattern XML_FIRST_BYTE = Pattern.compile("((.|\\s)*<\\?xml[^<]*)?\\s*<\\s*(\\w+:)?\\w+(.|\\s)*", Pattern.MULTILINE);
 
@@ -101,7 +101,7 @@ public static Optional<Path> downloadResource(URI resourceURL) {
       throw new MappingException("Error downloading resource from '" + resourceURL + "'!", tw);
     }
     downloadedFile = fixFileExtension(downloadedFile);
-
+    
     return Optional.ofNullable(downloadedFile);
   }
 
@@ -114,14 +114,11 @@ public static Optional<Path> downloadResource(URI resourceURL) {
   public static Path fixFileExtension(Path pathToFile) {
     Path returnFile = pathToFile;
     Path renamedFile = pathToFile;
+    LOGGER.trace("fixFileExtension({})", pathToFile);
     try {
       if ((pathToFile != null) && (pathToFile.toFile().exists())) {
-        Tika tika = new Tika();
-        String mimeType = tika.detect(pathToFile.toFile());
-        MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
-        MimeType estimatedMimeType = allTypes.forName(mimeType);
-        String newExtension = estimatedMimeType.getExtension(); // .jpg
-
+        String contentOfFile = FileUtils.readFileToString(pathToFile.toFile(), StandardCharsets.UTF_8);
+        String newExtension = guessFileExtension(contentOfFile.getBytes());
         if (newExtension != null) {
           if (!pathToFile.toString().endsWith(newExtension)) {
             renamedFile = Paths.get(pathToFile + newExtension);
@@ -130,9 +127,10 @@ public static Path fixFileExtension(Path pathToFile) {
           }
         }
       }
-    } catch (IOException|MimeTypeException ex) {
+    } catch (IOException ex) {
       LOGGER.error("Error moving file '{}' to '{}'.", pathToFile, renamedFile);
     }
+    LOGGER.trace("'{}' -> '{}'", pathToFile, returnFile);
     return returnFile;
   }
 
@@ -177,21 +175,38 @@ public static void removeFile(Path tempFile) {
    * @return Estimated extension. e.g. '.xml'
    */
   private static String guessFileExtension(byte[] schema) {
+    String returnValue = null;
     // Cut schema to a maximum of MAX_LENGTH_OF_HEADER characters.
     int length = Math.min(schema.length, MAX_LENGTH_OF_HEADER);
     String schemaAsString = new String(schema, 0, length);
     LOGGER.trace("Guess type for '{}'", schemaAsString);
-
+    
     Matcher m = JSON_FIRST_BYTE.matcher(schemaAsString);
     if (m.matches()) {
-      return ".json";
+      returnValue = ".json";
     } else {
       m = XML_FIRST_BYTE.matcher(schemaAsString);
       if (m.matches()) {
-        return ".xml";
+        returnValue = ".xml";
       }
     }
-    return null;
+    if (returnValue == null) {
+      // Use tika library to estimate extension
+      LOGGER.trace("Use tika library to estimate extension.");
+      Tika tika = new Tika();
+      String mimeType;
+      mimeType = tika.detect(schema);
+      MimeTypes allTypes = MimeTypes.getDefaultMimeTypes();
+      MimeType estimatedMimeType;
+      try {
+        estimatedMimeType = allTypes.forName(mimeType);
+        returnValue = estimatedMimeType.getExtension(); // .jpg
+        LOGGER.trace("Mimetype: '{}', Extension: '{}'", mimeType, returnValue);
+      } catch (MimeTypeException ex) {
+        LOGGER.error("Unknown mimetype '{}'", mimeType);
+      }
+    }
+    return returnValue;
   }
 
   /**
@@ -217,7 +232,7 @@ public static Path cloneGitRepository(String repositoryUrl, String branch) {
   public static Path cloneGitRepository(String repositoryUrl, String branch, String targetFolder) {
     File target = new File(targetFolder);
     target.mkdirs();
-
+    
     LOGGER.info("Cloning branch '{}' of repository '{}' to '{}'", branch, repositoryUrl, target.getPath());
     try {
       Git.cloneRepository().setURI(repositoryUrl).setBranch(branch).setDirectory(target).call();
@@ -226,7 +241,7 @@ public static Path cloneGitRepository(String repositoryUrl, String branch, Strin
     } catch (GitAPIException ex) {
       throw new MappingException("Error cloning git repository '" + repositoryUrl + "' to '" + target + "'!", ex);
     }
-
+    
     return target.toPath();
   }
 }

src/test/java/edu/kit/datamanager/mappingservice/util/FileUtilTest.java

@@ -142,11 +142,12 @@ public void testDownloadLocalXMLFileWithoutSuffix() throws IOException {
     public void testDownloadLocalResourceWithoutSuffix() {
         System.out.println("testDownloadLocalResource");
         File srcFile = new File("src/test/resources/examples/anyContentWithoutSuffix");
+        String expectedExtension = ".txt";
         assertTrue(srcFile.exists());
         Optional<Path> result = FileUtil.downloadResource(srcFile.getAbsoluteFile().toURI());
         assertTrue(result.isPresent());
         assertTrue(result.get().toFile().exists());
-        assertTrue(result.get().toString().endsWith(FileUtil.DEFAULT_SUFFIX));
+        assertTrue(result.get().toString().endsWith(expectedExtension));
         assertTrue(result.get().toFile().delete());
     }
 
@@ -249,7 +250,7 @@ public void testFixFileExtensionJson() throws IOException {
         System.out.println("testFixFileExtensionJson");
         File srcFile = new File("src/test/resources/examples/gemma/simple.json");
         assertTrue(srcFile.exists());
-        String[] extensions = {"nosuffix", "json", ".json ", ".xml"};
+        String[] extensions = {"nosuffix", "json", ".json", ".xml"};
         for (String extension : extensions) {
             Path createTempFile = FileUtil.createTempFile(null, extension);
             Files.copy(srcFile, createTempFile.toFile());
@@ -263,14 +264,34 @@ public void testFixFileExtensionJson() throws IOException {
     @Test
     public void testFixFileExtensionUnknown() throws IOException {
         System.out.println("testFixFileExtensionUnknown");
+
         File srcFile = new File("src/test/resources/examples/anyContentWithoutSuffix");
         assertTrue(srcFile.exists());
-        String[] extensions = {"nosuffix", "json", ".json ", ".xml"};
+        String[] extensions = {"nosuffix", "json", ".json", ".xml", ".txt"};
+        String expectedExtension = ".txt";
+        for (String extension : extensions) {
+            Path createTempFile = FileUtil.createTempFile(null, extension);
+            Files.copy(srcFile, createTempFile.toFile());
+            Path result = FileUtil.fixFileExtension(createTempFile);
+            assertTrue(result.toString().endsWith(expectedExtension), "Result: " + result.toString());
+            assertTrue(result.toFile().delete());
+
+        }
+    }
+
+    @Test
+    public void testFixZipFileExtensionUnknown() throws IOException {
+        System.out.println("testFixFileExtensionUnknown");
+
+        File srcFile = new File("src/test/resources/examples/record_json_zip");
+        assertTrue(srcFile.exists());
+        String[] extensions = {"nosuffix", ".zip", ".json", ".xml"};
+        String expectedExtension = ".zip";
         for (String extension : extensions) {
             Path createTempFile = FileUtil.createTempFile(null, extension);
             Files.copy(srcFile, createTempFile.toFile());
             Path result = FileUtil.fixFileExtension(createTempFile);
-            assertTrue(result.toString().endsWith(extension));
+            assertTrue(result.toString().endsWith(expectedExtension));
             assertTrue(result.toFile().delete());
 
         }