Skip to content

Commit e392547

Browse files
LucaCinquiniLucaCinquini
committed
Inserting support for uploading product-level metadata.
1 parent e44525b commit e392547

File tree

5 files changed

+113
-16
lines changed

5 files changed

+113
-16
lines changed

‎common/pom.xml

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,13 +58,31 @@
5858

5959
<!-- install jar file into OODT modules -->
6060
<execution>
61-
<id>copy-jar</id>
61+
<id>copy-jar-to-workflow</id>
6262
<phase>install</phase>
6363
<goals>
6464
<goal>copy-resources</goal>
6565
</goals>
6666
<configuration>
67-
<outputDirectory>${oodt.dir}/cas-workflow-${oodt.version}/lib</outputDirectory>
67+
<outputDirectory>${oodt.dir}/cas-workflow/lib</outputDirectory>
68+
<resources>
69+
<resource>
70+
<directory>${project.basedir}/target</directory>
71+
<includes>
72+
<include>${parent.artifactId}-${artifactId}-${version}.jar</include>
73+
</includes>
74+
</resource>
75+
</resources>
76+
</configuration>
77+
</execution>
78+
<execution>
79+
<id>copy-jar-to-filemgr</id>
80+
<phase>install</phase>
81+
<goals>
82+
<goal>copy-resources</goal>
83+
</goals>
84+
<configuration>
85+
<outputDirectory>${oodt.dir}/cas-filemgr/lib</outputDirectory>
6886
<resources>
6987
<resource>
7088
<directory>${project.basedir}/target</directory>

‎common/src/main/java/gov/nasa/jpl/edrn/labcas/Constants.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,15 +23,18 @@ public interface Constants {
2323
public final static String METADATA_KEY_DATASET = "Dataset";
2424
public final static String METADATA_KEY_VERSION = "Version";
2525
public final static String METADATA_KEY_DESCRIPTION = "Description";
26+
public final static String METADATA_KEY_FILE_LOCATION = "FileLocation";
27+
public final static String METADATA_KEY_FILE_NAME = "Filename";
2628

2729
// dataset-level metadata file
28-
public final static String METADATA_FILE = "DatasetMetadata.xml";
30+
public final static String METADATA_FILE = "DatasetMetadata.xmlmet";
2931

3032
// policy directory
3133
public final static String POLICY = "policy";
3234

33-
// OODT metadata extension
34-
public final static String METADATA_EXTENSION = ".met";
35+
// metadata file extensions
36+
public final static String OODT_METADATA_EXTENSION = ".met";
37+
public final static String EDRN_METADATA_EXTENSION = ".xmlmet";
3538

3639
// maximum number of records returned by Solr
3740
// (i.e. maximum number of files in a single dataset that can be updated)

‎common/src/main/java/gov/nasa/jpl/edrn/labcas/FileManagerUtils.java

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88
import java.util.List;
99
import java.util.logging.Logger;
1010

11-
import org.apache.commons.lang.WordUtils;
12-
import org.apache.oodt.cas.filemgr.cli.action.FileManagerCliAction;
1311
import org.apache.oodt.cas.filemgr.metadata.extractors.CoreMetExtractor;
1412
import org.apache.oodt.cas.filemgr.metadata.extractors.examples.MimeTypeExtractor;
1513
import org.apache.oodt.cas.filemgr.structs.ProductType;
@@ -20,6 +18,8 @@
2018
import org.w3c.dom.Document;
2119
import org.w3c.dom.Element;
2220

21+
import gov.nasa.jpl.edrn.labcas.extractors.XmlFileMetExtractor;
22+
2323
/**
2424
* Class that contains common functionality to interact with the FileManager.
2525
*
@@ -198,7 +198,7 @@ public boolean accept(File current, String name) {
198198

199199
/**
200200
* Utility method that reads the additional dataset metadata
201-
* from the file DatasetMetadata.xml located in the dataset staging directory.
201+
* from the file DatasetMetadata.xmlmet located in the dataset staging directory.
202202
*
203203
* @param datasetName
204204
* @return
@@ -208,19 +208,32 @@ public static Metadata readDatasetMetadata(final String datasetName) {
208208

209209
String stagingDir = System.getenv(Constants.ENV_LABCAS_STAGING) + "/" + datasetName;
210210
File datasetMetadataFile = new File(stagingDir, Constants.METADATA_FILE);
211+
212+
return readMetadata(datasetMetadataFile);
213+
214+
}
215+
216+
/**
217+
* Utility method to read metadata from a file.
218+
*
219+
* @param metadataFilepath
220+
* @return
221+
*/
222+
public static Metadata readMetadata(final File metadataFilepath) {
211223

212-
// read input metadata
224+
// read file metadata
213225
Metadata metadata = new Metadata(); // empty metadata container
214-
if (datasetMetadataFile.exists()) {
215-
LOG.info("Updating metadata from file: "+datasetMetadataFile.getAbsolutePath());
226+
227+
if (metadataFilepath.exists()) {
228+
LOG.info("Reading metadata from file: "+metadataFilepath.getAbsolutePath());
216229

217230
try {
218231
SerializableMetadata sm = new SerializableMetadata("UTF-8", false);
219-
sm.loadMetadataFromXmlStream(datasetMetadataFile.toURI().toURL().openStream());
232+
sm.loadMetadataFromXmlStream(metadataFilepath.toURI().toURL().openStream());
220233
metadata = sm.getMetadata();
221234
for (String key : metadata.getAllKeys()) {
222235
for (String val : metadata.getAllMetadata(key)) {
223-
LOG.fine("\t==> Read dataset metadata key=["+key+"] value=["+val+"]");
236+
LOG.fine("\t==> Read metadata key=["+key+"] value=["+val+"]");
224237
}
225238
}
226239

@@ -229,7 +242,7 @@ public static Metadata readDatasetMetadata(final String datasetName) {
229242
}
230243

231244
} else {
232-
LOG.warning("Metadata file: "+datasetMetadataFile.getAbsolutePath()+" not found");
245+
LOG.warning("Metadata file: "+metadataFilepath.getAbsolutePath()+" not found");
233246
}
234247

235248
return metadata;
@@ -387,6 +400,16 @@ public static final void makeProductTypesXmlFile(File filepath, String productTy
387400
// <configuration>
388401
Element configuration2Element = xmlDocument.createElement("configuration");
389402
extractor2Element.appendChild(configuration2Element);
403+
404+
// <extractor class="gov.nasa.jpl.edrn.labcas.extractors.XmlFileMetExtractor">
405+
Element extractor3Element = xmlDocument.createElement("extractor");
406+
extractor3Element.setAttribute("class", XmlFileMetExtractor.class.getCanonicalName());
407+
metExtractorsElement.appendChild(extractor3Element);
408+
409+
// <configuration>
410+
Element configuration3Element = xmlDocument.createElement("configuration");
411+
extractor3Element.appendChild(configuration3Element);
412+
390413

391414
// <metadata>
392415
Element metadataElement = xmlDocument.createElement("metadata");

‎common/src/main/java/gov/nasa/jpl/edrn/labcas/extractors/XmlFileMetExtractor.java

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
package gov.nasa.jpl.edrn.labcas.extractors;
2+
3+
import java.io.File;
4+
import java.util.Properties;
5+
import java.util.logging.Logger;
6+
7+
import org.apache.oodt.cas.filemgr.metadata.extractors.FilemgrMetExtractor;
8+
import org.apache.oodt.cas.filemgr.structs.Product;
9+
import org.apache.oodt.cas.metadata.Metadata;
10+
import org.apache.oodt.cas.metadata.exceptions.MetExtractionException;
11+
12+
import gov.nasa.jpl.edrn.labcas.Constants;
13+
import gov.nasa.jpl.edrn.labcas.FileManagerUtils;
14+
15+
/**
16+
* Class that reads product-level metadata from an XML file
17+
* with the same name in the same directory.
18+
*
19+
* @author luca
20+
*
21+
*/
22+
public class XmlFileMetExtractor implements FilemgrMetExtractor {
23+
24+
private static final Logger LOG = Logger.getLogger(XmlFileMetExtractor.class.getName());
25+
26+
@Override
27+
public void configure(Properties properties) {}
28+
29+
@Override
30+
public Metadata extractMetadata(Product product, Metadata inmet) throws MetExtractionException {
31+
32+
LOG.info("Running XmlFileMetExtractor for product:" + product.getProductName());
33+
34+
// merge original metadata
35+
Metadata outmet = new Metadata();
36+
outmet.addMetadata(inmet.getHashtable());
37+
38+
// construct path to optional metadata file
39+
File xmlmetFilepath = new File(inmet.getMetadata(Constants.METADATA_KEY_FILE_LOCATION),
40+
inmet.getMetadata(Constants.METADATA_KEY_FILE_NAME)+Constants.EDRN_METADATA_EXTENSION);
41+
42+
// add metadata from file
43+
if (xmlmetFilepath.exists()) {
44+
LOG.info("Adding additional product metadata from file: "+xmlmetFilepath.getAbsolutePath());
45+
46+
Metadata fmet = FileManagerUtils.readMetadata(xmlmetFilepath);
47+
outmet.addMetadata(fmet.getHashtable());
48+
49+
}
50+
51+
return outmet;
52+
}
53+
54+
}

‎common/src/main/java/gov/nasa/jpl/edrn/labcas/tasks/LabcasUploadInitTaskInstance.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@
1111

1212
import gov.nasa.jpl.edrn.labcas.Constants;
1313
import gov.nasa.jpl.edrn.labcas.FileManagerUtils;
14-
import gov.nasa.jpl.edrn.labcas.XmlUtils;
1514

1615
/**
1716
* Task used to initialize a LabCAS crawler workflow.
@@ -60,7 +59,7 @@ public void run(Metadata metadata, WorkflowTaskConfiguration config) throws Work
6059
String[] metFiles = new File(stagingDir).list(new FilenameFilter() {
6160
@Override
6261
public boolean accept(File current, String name) {
63-
return new File(current, name).getAbsolutePath().endsWith(Constants.METADATA_EXTENSION);
62+
return new File(current, name).getAbsolutePath().endsWith(Constants.OODT_METADATA_EXTENSION);
6463
}
6564
});
6665
for (String metFile : metFiles) {

0 commit comments

Comments
 (0)