Skip to content

Commit 7eac095

Browse files
LucaCinquiniLucaCinquini
committed
Working on update workflow.
1 parent ed02fc5 commit 7eac095

File tree

7 files changed

+456
-199
lines changed

7 files changed

+456
-199
lines changed

‎INSTALL.txt

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,11 +84,19 @@ cd $LABCAS_HOME/cas-workflow/bin
8484

8585
./wmgr-client --url http://localhost:9001 --operation --sendEvent --eventName labcas-test --metaData --key experiment 11 --key species snakes
8686

87+
88+
8789
o Trigger crawling for uploaded dataset "mydata", stored in directory $LABCAS_STAGIN/mydata
8890

8991
./wmgr-client --url http://localhost:9001 --operation --sendEvent --eventName labcas-upload --metaData --key Dataset mydata
9092

91-
will also ingest metadata contained in file $LABCAS_STAGIN/mydata/DatasetMetadata.xmlS
93+
will also ingest metadata contained in file $LABCAS_STAGIN/mydata/DatasetMetadata.xml
94+
95+
NOTE: every time the labcas-workflow is run:
96+
- a new dataset version will be uploaded (even if nothing changed)
97+
- the new metadata will completely override the previous metadata
98+
99+
To update metadata without changing the version, use the "labcas-upadte" workflow.
92100

93101
o Trigger metadata update for previously uploaded dataset "mydata":
94102

‎common/src/main/java/gov/nasa/jpl/edrn/labcas/Constants.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
public interface Constants {
1010

1111
// environment variables
12+
public final static String ENV_LABCAS_HOME = "LABCAS_HOME";
1213
public final static String ENV_LABCAS_STAGING = "LABCAS_STAGING";
1314
public final static String ENV_LABCAS_ARCHIVE = "LABCAS_ARCHIVE";
1415

‎common/src/main/java/gov/nasa/jpl/edrn/labcas/FileManagerUtils.java

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
package gov.nasa.jpl.edrn.labcas;
2+
3+
import java.io.File;
4+
import java.io.IOException;
5+
import java.net.URL;
6+
import java.util.ArrayList;
7+
import java.util.Arrays;
8+
import java.util.List;
9+
import java.util.logging.Logger;
10+
11+
import org.apache.commons.lang.WordUtils;
12+
import org.apache.oodt.cas.filemgr.structs.ProductType;
13+
import org.apache.oodt.cas.filemgr.system.XmlRpcFileManagerClient;
14+
import org.apache.oodt.cas.filemgr.util.XmlStructFactory;
15+
import org.apache.oodt.cas.metadata.Metadata;
16+
import org.apache.oodt.cas.metadata.SerializableMetadata;
17+
import gov.nasa.jpl.edrn.labcas.Constants;
18+
19+
/**
20+
* Class that contains common functionality to interact with the FileManager
21+
* both through its XML/RPC interface (for dynamic updates)
22+
* and its XML repository (for persisting the updates to XML).
23+
*
24+
* @author luca
25+
*
26+
*/
27+
public class FileManagerUtils {
28+
29+
// default value for FileManager URL
30+
private static String FILEMANAGER_URL = "http://localhost:9000/";
31+
static {
32+
if (System.getenv(Constants.ENV_FILEMGR_URL)!=null) {
33+
FILEMANAGER_URL = System.getenv(Constants.ENV_FILEMGR_URL);
34+
}
35+
}
36+
37+
private static final Logger LOG = Logger.getLogger(FileManagerUtils.class.getName());
38+
39+
/**
40+
* Method to add a new dataset, or update an existing dataset, into the File Manager.
41+
*
42+
* @param dataset
43+
*/
44+
public static void updateDataset(String dataset) throws Exception {
45+
46+
XmlRpcFileManagerClient client = new XmlRpcFileManagerClient(new URL(FILEMANAGER_URL));
47+
48+
// build product type from dataset
49+
String productTypeName = FileManagerUtils.getProductTypeName(dataset);
50+
51+
// retrieve additional dataset metadata from file DatasetMetadata.xml
52+
Metadata datasetMetadata = FileManagerUtils.readDatasetMetadata( dataset );
53+
54+
55+
// transfer metadata field 'Description' to dataset description
56+
String datasetDescription = dataset; // default
57+
if (datasetMetadata.containsKey(Constants.METADATA_KEY_DESCRIPTION)) {
58+
datasetDescription = datasetMetadata.getMetadata(Constants.METADATA_KEY_DESCRIPTION);
59+
datasetMetadata.removeMetadata(Constants.METADATA_KEY_DESCRIPTION);
60+
}
61+
62+
// query File Manager for product type object
63+
ProductType productType = client.getProductTypeByName(productTypeName);
64+
LOG.info("Retrieved product type id="+productType.getProductTypeId()+" from File Manager");
65+
66+
// update the product type object
67+
//productType.setDescription(datasetDescription);
68+
69+
// loop over all metadata keys, values found in DatasetMetadata.xml
70+
Metadata typeMetadata = productType.getTypeMetadata();
71+
for (String key : datasetMetadata.getAllKeys()) {
72+
LOG.info("Updating key="+key);
73+
74+
// remove this metadata key
75+
typeMetadata.removeMetadata(key);
76+
77+
// insert all new (non-empty) values, overriding old values
78+
List<String> vals = new ArrayList<String>();
79+
for (String value : datasetMetadata.getAllMetadata(key)) {
80+
if (value.trim().length()>0) {
81+
LOG.info("Adding value="+value+" to key="+key);
82+
typeMetadata.addMetadata(key, value);
83+
}
84+
}
85+
86+
}
87+
88+
// write the updated product type object to XML
89+
final List<ProductType> producTypes = Arrays.asList( new ProductType[] { productType });
90+
for (ProductType pt : producTypes) {
91+
LOG.info("Updating product type="+pt.getName());
92+
}
93+
File productTypesXmlFile = new File(FileManagerUtils.getDatasetDir(dataset), "/policy/product-types.xml");
94+
XmlStructFactory.writeProductTypeXmlDocument(producTypes, productTypesXmlFile.getAbsolutePath());
95+
LOG.info("Written XML file="+ productTypesXmlFile.getAbsolutePath());
96+
97+
// send updated object to the File Manager
98+
String productTypeId = client.addProductType(productType);
99+
LOG.info("Updated product type="+productTypeId);
100+
101+
}
102+
103+
104+
/**
105+
* Utility method that reads the additional dataset metadata
106+
* from the file DatasetMetadata.xml located in the dataset staging directory.
107+
*
108+
* @param datasetName
109+
* @return
110+
* @throws IOException
111+
*/
112+
private static Metadata readDatasetMetadata(final String datasetName) {
113+
114+
String stagingDir = System.getenv(Constants.ENV_LABCAS_STAGING) + "/" + datasetName;
115+
File datasetMetadataFile = new File(stagingDir, Constants.METADATA_FILE);
116+
117+
// read input metadata
118+
Metadata metadata = new Metadata(); // empty metadata container
119+
if (datasetMetadataFile.exists()) {
120+
LOG.info("Updating metadata from file: "+datasetMetadataFile.getAbsolutePath());
121+
122+
try {
123+
SerializableMetadata sm = new SerializableMetadata("UTF-8", false);
124+
sm.loadMetadataFromXmlStream(datasetMetadataFile.toURI().toURL().openStream());
125+
metadata = sm.getMetadata();
126+
for (String key : metadata.getAllKeys()) {
127+
for (String val : metadata.getAllMetadata(key)) {
128+
LOG.fine("\t==> Read dataset metadata key=["+key+"] value=["+val+"]");
129+
}
130+
}
131+
132+
} catch (Exception e) {
133+
LOG.warning(e.getMessage());
134+
}
135+
136+
} else {
137+
LOG.warning("Metadata file: "+datasetMetadataFile.getAbsolutePath()+" not found");
138+
}
139+
140+
return metadata;
141+
142+
}
143+
144+
/**
145+
* Retrieves the directory where an uploaded dataset will be archived.
146+
*
147+
* @param datasetName
148+
* @return
149+
*/
150+
private static File getDatasetDir(final String datasetName) {
151+
152+
String archiveDir = System.getenv(Constants.ENV_LABCAS_ARCHIVE) + "/" + Constants.WORKFLOW_LABCAS_UPOLOAD;
153+
File datasetDir = new File(archiveDir, datasetName);
154+
return datasetDir;
155+
156+
}
157+
158+
/**
159+
* Constructs the product type name from a dataset identifier.
160+
* @param dataset
161+
* @return
162+
*/
163+
private static String getProductTypeName(String dataset) {
164+
165+
String productType = WordUtils.capitalize(dataset).replaceAll("\\s+", "_");
166+
return productType;
167+
168+
}
169+
170+
}

‎common/src/main/java/gov/nasa/jpl/edrn/labcas/Utils.java

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import javax.xml.transform.stream.StreamResult;
2121

2222
import org.apache.commons.io.FileUtils;
23+
import org.apache.commons.lang.WordUtils;
2324
import org.apache.oodt.cas.filemgr.metadata.extractors.CoreMetExtractor;
2425
import org.apache.oodt.cas.filemgr.metadata.extractors.examples.MimeTypeExtractor;
2526
import org.apache.oodt.cas.filemgr.structs.ExtractorSpec;
@@ -88,6 +89,18 @@ public static File getDatasetDir(final String datasetName) {
8889

8990
}
9091

92+
/**
93+
* Constructs the product type name from a dataset identifier.
94+
* @param dataset
95+
* @return
96+
*/
97+
public static String getProductTypeName(String dataset) {
98+
99+
String productType = WordUtils.capitalize(dataset).replaceAll("\\s+", "_");
100+
return productType;
101+
102+
}
103+
91104
/**
92105
* Utility method that reads the additional dataset metadata
93106
* from the file DatasetMetadata.xml located in the dataset staging directory.
@@ -220,6 +233,7 @@ public static final void makeProductTypeElementMapXmlFile(File filepath, String
220233

221234
}
222235

236+
223237
/**
224238
* Utility method to create the file "product-types.xml".
225239
* @param filepath
@@ -330,6 +344,27 @@ public static final void makeProductTypesXmlFile(File filepath, String productTy
330344

331345
}
332346

347+
/**
348+
* Method to retrieve a ProductType from the File Manager XML/RPC interface.
349+
*
350+
* @param productTypeName
351+
*/
352+
/**
353+
public final static ProductType getProductType(String productTypeName) throws Exception {
354+
355+
// instantiate XML/RPC client to File Manager
356+
String fmURL = System.getenv(Constants.ENV_FILEMGR_URL);
357+
if (fmURL==null) {
358+
fmURL = "http://localhost:9000/";
359+
}
360+
XmlRpcFileManagerClient client = new XmlRpcFileManagerClient(new URL(fmURL));
361+
362+
ProductType productType = client.getProductTypeByName(productTypeName);
363+
364+
return productType;
365+
366+
}*/
367+
333368
/**
334369
* Method that adds or updates a ProductType through the File Manager XML/RPC interface.
335370
*/

0 commit comments

Comments
 (0)