Skip to content
This repository was archived by the owner on May 24, 2022. It is now read-only.

Commit 7530e8b

Browse files
knutwalkerknutwalker
committed
Merge pull request #20 from zazi/sprint-40/dd-999
[DD-999] data model deprecation endpoint
2 parents 44d08de + e79d458 commit 7530e8b

14 files changed

+859
-200
lines changed

‎src/main/java/org/dswarm/graph/BasicNeo4jProcessor.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.dswarm.graph.model.Statement;
4848
import org.dswarm.graph.tx.TransactionHandler;
4949
import org.dswarm.graph.utils.GraphDatabaseUtils;
50+
import org.dswarm.graph.utils.GraphUtils;
5051
import org.dswarm.graph.versioning.VersionHandler;
5152

5253
/**
@@ -448,6 +449,17 @@ public Relationship prepareRelationship(final Node subjectNode, final String pre
448449
return rel;
449450
}
450451

452+
public long generateStatementHash(final Relationship rel) throws DMPGraphException {
453+
454+
final Node subjectNode = rel.getStartNode();
455+
final Node objectNode = rel.getEndNode();
456+
final String predicateName = rel.getType().name();
457+
final NodeType subjectNodeType = GraphUtils.determineNodeType(subjectNode);
458+
final NodeType objectNodeType = GraphUtils.determineNodeType(objectNode);
459+
460+
return generateStatementHash(subjectNode, predicateName, objectNode, subjectNodeType, objectNodeType);
461+
}
462+
451463
public long generateStatementHash(final Node subjectNode, final String predicateName, final Node objectNode, final NodeType subjectNodeType,
452464
final NodeType objectNodeType) throws DMPGraphException {
453465

‎src/main/java/org/dswarm/graph/deprecate/BaseNeo4jDeprecator.java‎

Lines changed: 221 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
/**
2+
* This file is part of d:swarm graph extension.
3+
*
4+
* d:swarm graph extension is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* d:swarm graph extension is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with d:swarm graph extension. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
package org.dswarm.graph.deprecate;
18+
19+
import org.neo4j.graphdb.Direction;
20+
import org.neo4j.graphdb.Node;
21+
import org.neo4j.graphdb.Relationship;
22+
import org.slf4j.Logger;
23+
import org.slf4j.LoggerFactory;
24+
25+
import org.dswarm.graph.BasicNeo4jProcessor;
26+
import org.dswarm.graph.DMPGraphException;
27+
import org.dswarm.graph.Neo4jProcessor;
28+
import org.dswarm.graph.NodeType;
29+
import org.dswarm.graph.model.GraphStatics;
30+
import org.dswarm.graph.read.NodeHandler;
31+
import org.dswarm.graph.read.RelationshipHandler;
32+
import org.dswarm.graph.utils.GraphUtils;
33+
import org.dswarm.graph.versioning.Range;
34+
import org.dswarm.graph.versioning.VersionHandler;
35+
import org.dswarm.graph.versioning.VersioningStatics;
36+
37+
/**
38+
* @author tgaengler
39+
*/
40+
public abstract class BaseNeo4jDeprecator implements RelationshipDeprecator {
41+
42+
private static final Logger LOG = LoggerFactory.getLogger(BaseNeo4jDeprecator.class);
43+
44+
protected final NodeHandler nodeHandler;
45+
protected final NodeHandler startNodeHandler;
46+
protected final RelationshipHandler relationshipHandler;
47+
48+
protected int i = 0;
49+
50+
protected VersionHandler versionHandler = null;
51+
protected int version;
52+
protected int previousVersion;
53+
54+
protected final BasicNeo4jProcessor processor;
55+
protected final boolean enableVersioning;
56+
57+
public BaseNeo4jDeprecator(final BasicNeo4jProcessor processorArg, final boolean enableVersioningArg) throws DMPGraphException {
58+
59+
processor = processorArg;
60+
enableVersioning = enableVersioningArg;
61+
62+
init();
63+
64+
nodeHandler = new CBDNodeHandler();
65+
startNodeHandler = new CBDStartNodeHandler();
66+
relationshipHandler = new CBDRelationshipHandler();
67+
}
68+
69+
@Override
70+
public Neo4jProcessor getProcessor() {
71+
72+
return processor;
73+
}
74+
75+
/**
76+
* tx should be running, i.e., no check is done atm
77+
*
78+
* @param rel
79+
* @throws DMPGraphException
80+
*/
81+
@Override
82+
public void deprecateStatement(final Relationship rel) throws DMPGraphException {
83+
84+
i++;
85+
86+
///processor.ensureRunningTx();
87+
88+
try {
89+
90+
rel.setProperty(VersioningStatics.VALID_TO_PROPERTY, version);
91+
final Long hashedUUID = (Long) rel.getProperty(GraphStatics.UUID_PROPERTY, null);
92+
93+
if (hashedUUID == null) {
94+
95+
LOG.debug("statement/relationship '{}' has no hashed UUID", rel.getId());
96+
97+
return;
98+
}
99+
100+
// remove statement hash from statement hashes index
101+
final long statementHash = processor.generateStatementHash(rel);
102+
processor.removeHashFromStatementIndex(statementHash);
103+
} catch (final Exception e) {
104+
105+
final String message = "couldn't deprecate statement successfully";
106+
107+
processor.failTx();
108+
109+
BaseNeo4jDeprecator.LOG.error(message, e);
110+
BaseNeo4jDeprecator.LOG.debug("couldn't finish write TX successfully");
111+
112+
throw new DMPGraphException(message);
113+
}
114+
}
115+
116+
@Override
117+
public VersionHandler getVersionHandler() {
118+
119+
return versionHandler;
120+
}
121+
122+
@Override
123+
public int getRelationshipsDeprecated() {
124+
125+
return i;
126+
}
127+
128+
@Override
129+
public void closeTransaction() throws DMPGraphException {
130+
131+
LOG.debug("close write TX finally");
132+
133+
processor.succeedTx();
134+
processor.clearMaps();
135+
}
136+
137+
protected abstract void init() throws DMPGraphException;
138+
139+
protected class CBDNodeHandler implements NodeHandler {
140+
141+
@Override
142+
public void handleNode(final Node node) throws DMPGraphException {
143+
144+
// TODO: find a better way to determine the end of a resource description, e.g., add a property "resource" to each
145+
// node that holds the uri of the resource (record)
146+
// => maybe we should find an appropriated cypher query as replacement for this processing
147+
if (!node.hasProperty(GraphStatics.URI_PROPERTY)) {
148+
149+
final Iterable<Relationship> relationships = node.getRelationships(Direction.OUTGOING);
150+
151+
for (final Relationship relationship : relationships) {
152+
153+
final Integer validFrom = (Integer) relationship.getProperty(VersioningStatics.VALID_FROM_PROPERTY, null);
154+
final Integer validTo = (Integer) relationship.getProperty(VersioningStatics.VALID_TO_PROPERTY, null);
155+
156+
if (validFrom != null && validTo != null) {
157+
158+
if (Range.range(validFrom, validTo).contains(previousVersion)) {
159+
160+
relationshipHandler.handleRelationship(relationship);
161+
}
162+
} else {
163+
164+
// TODO: remove this later, when every stmt is versioned
165+
relationshipHandler.handleRelationship(relationship);
166+
}
167+
}
168+
}
169+
}
170+
}
171+
172+
protected class CBDStartNodeHandler implements NodeHandler {
173+
174+
@Override
175+
public void handleNode(final Node node) throws DMPGraphException {
176+
177+
// TODO: find a better way to determine the end of a resource description, e.g., add a property "resource" to each
178+
// (this is the case for model that came as GDM JSON)
179+
// node that holds the uri of the resource (record)
180+
if (node.hasProperty(GraphStatics.URI_PROPERTY)) {
181+
182+
final Iterable<Relationship> relationships = node.getRelationships(Direction.OUTGOING);
183+
184+
for (final Relationship relationship : relationships) {
185+
186+
final Integer validFrom = (Integer) relationship.getProperty(VersioningStatics.VALID_FROM_PROPERTY, null);
187+
final Integer validTo = (Integer) relationship.getProperty(VersioningStatics.VALID_TO_PROPERTY, null);
188+
189+
if (validFrom != null && validTo != null) {
190+
191+
if (Range.range(validFrom, validTo).contains(previousVersion)) {
192+
193+
relationshipHandler.handleRelationship(relationship);
194+
}
195+
} else {
196+
197+
// TODO: remove this later, when every stmt is versioned
198+
relationshipHandler.handleRelationship(relationship);
199+
}
200+
}
201+
}
202+
}
203+
}
204+
205+
private class CBDRelationshipHandler implements RelationshipHandler {
206+
207+
@Override
208+
public void handleRelationship(final Relationship rel) throws DMPGraphException {
209+
210+
deprecateStatement(rel);
211+
212+
final Node objectNode = rel.getEndNode();
213+
final NodeType objectNodeType = GraphUtils.determineNodeType(objectNode);
214+
215+
if(!objectNodeType.equals(NodeType.Literal)) {
216+
217+
nodeHandler.handleNode(objectNode);
218+
}
219+
}
220+
}
221+
}

‎src/main/java/org/dswarm/graph/deprecate/DataModelNeo4jDeprecator.java‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
/**
2+
* This file is part of d:swarm graph extension.
3+
*
4+
* d:swarm graph extension is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* d:swarm graph extension is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with d:swarm graph extension. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
package org.dswarm.graph.deprecate;
18+
19+
import org.neo4j.graphdb.Node;
20+
import org.neo4j.graphdb.ResourceIterator;
21+
import org.slf4j.Logger;
22+
import org.slf4j.LoggerFactory;
23+
24+
import org.dswarm.graph.BasicNeo4jProcessor;
25+
import org.dswarm.graph.DMPGraphException;
26+
import org.dswarm.graph.GraphProcessingStatics;
27+
import org.dswarm.graph.model.GraphStatics;
28+
import org.dswarm.graph.versioning.DataModelNeo4jVersionHandler;
29+
30+
/**
31+
* @author tgaengler
32+
*/
33+
public class DataModelNeo4jDeprecator extends BaseNeo4jDeprecator {
34+
35+
private static final Logger LOG = LoggerFactory.getLogger(DataModelNeo4jDeprecator.class);
36+
37+
protected final String prefixedDataModelUri;
38+
39+
public DataModelNeo4jDeprecator(final BasicNeo4jProcessor processorArg, final boolean enableVersioningArg, final String prefixedDataModelUriArg)
40+
throws DMPGraphException {
41+
42+
super(processorArg, enableVersioningArg);
43+
44+
prefixedDataModelUri = prefixedDataModelUriArg;
45+
}
46+
47+
@Override
48+
protected void init() throws DMPGraphException {
49+
50+
versionHandler = new DataModelNeo4jVersionHandler(processor, enableVersioning);
51+
version = versionHandler.getLatestVersion();
52+
previousVersion = version - 1;
53+
}
54+
55+
@Override public void work() throws DMPGraphException {
56+
57+
LOG.debug("try to deprecate all statements for data model '{}'", prefixedDataModelUri);
58+
59+
processor.ensureRunningTx();
60+
61+
final ResourceIterator<Node> seedNodes = processor.getDatabase()
62+
.findNodes(GraphProcessingStatics.RESOURCE_LABEL, GraphStatics.DATA_MODEL_PROPERTY, prefixedDataModelUri);
63+
64+
if (seedNodes == null) {
65+
66+
LOG.debug("there are no nodes for data model '{}' in the graph", prefixedDataModelUri);
67+
68+
return;
69+
}
70+
71+
if (!seedNodes.hasNext()) {
72+
73+
LOG.debug("there are no nodes for data model '{}' in the graph", prefixedDataModelUri);
74+
}
75+
76+
while (seedNodes.hasNext()) {
77+
78+
final Node seedNode = seedNodes.next();
79+
80+
startNodeHandler.handleNode(seedNode);
81+
}
82+
83+
LOG.debug("finished deprecating all statements for data model '{}'", prefixedDataModelUri);
84+
}
85+
}

‎src/main/java/org/dswarm/graph/deprecate/RelationshipDeprecator.java‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
/**
2+
* This file is part of d:swarm graph extension.
3+
*
4+
* d:swarm graph extension is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* d:swarm graph extension is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with d:swarm graph extension. If not, see <http://www.gnu.org/licenses/>.
16+
*/
17+
package org.dswarm.graph.deprecate;
18+
19+
import org.neo4j.graphdb.Relationship;
20+
21+
import org.dswarm.graph.DMPGraphException;
22+
import org.dswarm.graph.Neo4jProcessor;
23+
import org.dswarm.graph.versioning.VersionHandler;
24+
25+
/**
26+
* @author tgaengler
27+
*/
28+
public interface RelationshipDeprecator {
29+
30+
Neo4jProcessor getProcessor();
31+
32+
void work() throws DMPGraphException;
33+
34+
void deprecateStatement(final Relationship relationship) throws DMPGraphException;
35+
36+
VersionHandler getVersionHandler();
37+
38+
int getRelationshipsDeprecated();
39+
40+
void closeTransaction() throws DMPGraphException;
41+
}

‎src/main/java/org/dswarm/graph/parse/BaseNeo4jHandler.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,8 @@
2828
import java.util.UUID;
2929
import java.util.concurrent.atomic.AtomicLong;
3030

31-
import com.github.emboss.siphash.SipHash;
32-
import com.google.common.base.Charsets;
3331
import com.google.common.base.Optional;
3432
import com.hp.hpl.jena.vocabulary.RDF;
35-
import org.apache.commons.lang.NotImplementedException;
3633
import org.neo4j.graphdb.Label;
3734
import org.neo4j.graphdb.Node;
3835
import org.neo4j.graphdb.Relationship;
@@ -407,7 +404,9 @@ public Relationship deprecateStatement(final Long uuid) throws DMPGraphException
407404

408405
rel.setProperty(VersioningStatics.VALID_TO_PROPERTY, versionHandler.getLatestVersion());
409406

410-
// TODO: remove statement hash from statement hashes index
407+
// remove statement hash from statement hashes index
408+
final long statementHash = processor.generateStatementHash(rel);
409+
processor.removeHashFromStatementIndex(statementHash);
411410

412411
return rel;
413412
} catch (final Exception e) {

0 commit comments

Comments
 (0)