Skip to content

Commit 6328613

Browse files
jorgeepditommaso
jorgee
and
pditommaso
authored
Add version, kind, and spec to lineage schema (#6075) [ci fast]
Signed-off-by: jorgee <jorge.ejarque@seqera.io> Signed-off-by: Jorge Ejarque <jorgee@users.noreply.github.com> Signed-off-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com> Co-authored-by: Paolo Di Tommaso <paolo.ditommaso@gmail.com>
1 parent bec4bf7 commit 6328613

File tree

7 files changed

+254
-59
lines changed

7 files changed

+254
-59
lines changed

‎modules/nf-commons/src/main/nextflow/serde/gson/RuntimeTypeAdapterFactory.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,18 @@ public RuntimeTypeAdapterFactory<T> registerSubtype(Class<? extends T> type) {
251251
return registerSubtype(type, type.getSimpleName());
252252
}
253253

254+
protected Class<?> getSubTypeFromLabel(String label){
255+
return labelToSubtype.get(label);
256+
}
257+
258+
protected String getLabelFromSubtype(Class<?> subType){
259+
return subtypeToLabel.get(subType);
260+
}
261+
262+
protected String getTypeFieldName(){
263+
return typeFieldName;
264+
}
265+
254266
@Override
255267
public <R> TypeAdapter<R> create(Gson gson, TypeToken<R> type) {
256268
if (type == null) {

‎modules/nf-lineage/src/main/nextflow/lineage/serde/LinTypeAdapterFactory.groovy

Lines changed: 31 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,13 @@
1616
package nextflow.lineage.serde
1717

1818
import com.google.gson.Gson
19-
import com.google.gson.JsonElement
2019
import com.google.gson.JsonObject
2120
import com.google.gson.JsonParseException
2221
import com.google.gson.JsonParser
2322
import com.google.gson.TypeAdapter
2423
import com.google.gson.reflect.TypeToken
2524
import com.google.gson.stream.JsonReader
2625
import com.google.gson.stream.JsonWriter
27-
2826
import groovy.transform.CompileStatic
2927
import nextflow.lineage.model.v1beta1.FileOutput
3028
import nextflow.lineage.model.v1beta1.LinModel
@@ -34,15 +32,16 @@ import nextflow.lineage.model.v1beta1.Workflow
3432
import nextflow.lineage.model.v1beta1.WorkflowOutput
3533
import nextflow.lineage.model.v1beta1.WorkflowRun
3634
import nextflow.serde.gson.RuntimeTypeAdapterFactory
37-
3835
/**
3936
* Class to serialize LiSerializable objects including the Lineage model version.
4037
*
4138
* @author Jorge Ejarque <jorge.ejarque@seqera.io>
4239
*/
4340
@CompileStatic
4441
class LinTypeAdapterFactory<T> extends RuntimeTypeAdapterFactory<T> {
42+
4543
public static final String VERSION_FIELD = 'version'
44+
public static final String SPEC_FIELD = 'spec'
4645
public static final String CURRENT_VERSION = LinModel.VERSION
4746

4847
LinTypeAdapterFactory() {
@@ -53,7 +52,6 @@ class LinTypeAdapterFactory<T> extends RuntimeTypeAdapterFactory<T> {
5352
.registerSubtype(TaskRun, TaskRun.simpleName)
5453
.registerSubtype(TaskOutput, TaskOutput.simpleName)
5554
.registerSubtype(FileOutput, FileOutput.simpleName)
56-
5755
}
5856

5957
@Override
@@ -70,39 +68,43 @@ class LinTypeAdapterFactory<T> extends RuntimeTypeAdapterFactory<T> {
7068
return new TypeAdapter<R>() {
7169
@Override
7270
void write(JsonWriter out, R value) throws IOException {
73-
def json = delegate.toJsonTree(value)
74-
if (json instanceof JsonObject) {
75-
json = addVersion(json)
76-
}
77-
gson.toJson(json, out)
71+
final object = new JsonObject()
72+
object.addProperty(VERSION_FIELD, CURRENT_VERSION)
73+
String label = getLabelFromSubtype(value.class)
74+
if (!label)
75+
throw new JsonParseException("Not registered class ${value.class}")
76+
object.addProperty(getTypeFieldName(), label)
77+
def json = gson.toJsonTree(value)
78+
object.add(SPEC_FIELD, json)
79+
gson.toJson(object, out)
7880
}
7981

8082
@Override
8183
R read(JsonReader reader) throws IOException {
82-
def json = JsonParser.parseReader(reader)
83-
if (json instanceof JsonObject) {
84-
def obj = (JsonObject) json
85-
def versionEl = obj.get(VERSION_FIELD)
86-
if (versionEl == null || versionEl.asString != CURRENT_VERSION) {
87-
throw new JsonParseException("Invalid or missing version")
88-
}
84+
final obj = JsonParser.parseReader(reader)?.getAsJsonObject()
85+
if( obj==null )
86+
throw new JsonParseException("Parsed JSON object is null")
87+
final versionEl = obj.get(VERSION_FIELD)
88+
if (versionEl == null || versionEl.asString != CURRENT_VERSION) {
89+
throw new JsonParseException("Invalid or missing '${VERSION_FIELD}' JSON property")
90+
}
91+
final typeEl = obj.get(getTypeFieldName())
92+
if( typeEl==null )
93+
throw new JsonParseException("JSON property '${getTypeFieldName()}' not found")
94+
95+
// Check if this is the new format (has 'spec' field) or old format (data at root level)
96+
final specEl = obj.get(SPEC_FIELD)?.asJsonObject
97+
if ( specEl != null ) {
98+
// New format: data is wrapped in 'spec' field
99+
specEl.add(getTypeFieldName(), typeEl)
100+
return (R) delegate.fromJsonTree(specEl)
101+
} else {
102+
// Old format: data is at root level, just remove version field
89103
obj.remove(VERSION_FIELD)
104+
return (R) delegate.fromJsonTree(obj)
90105
}
91-
return delegate.fromJsonTree(json)
92106
}
93107
}
94108
}
95109

96-
private static JsonObject addVersion(JsonObject json){
97-
if( json.has(VERSION_FIELD) )
98-
throw new JsonParseException("object already defines a field named ${VERSION_FIELD}")
99-
100-
JsonObject clone = new JsonObject();
101-
clone.addProperty(VERSION_FIELD, CURRENT_VERSION)
102-
for (Map.Entry<String, JsonElement> e : json.entrySet()) {
103-
clone.add(e.getKey(), e.getValue());
104-
}
105-
return clone
106-
}
107-
108110
}

‎modules/nf-lineage/src/test/nextflow/lineage/cli/LinCommandImplTest.groovy

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -387,27 +387,27 @@ class LinCommandImplTest extends Specification{
387387
def expectedOutput = '''diff --git 12345 67890
388388
--- 12345
389389
+++ 67890
390-
@@ -1,16 +1,16 @@
391-
{
390+
@@ -2,16 +2,16 @@
392391
"version": "lineage/v1beta1",
393392
"kind": "FileOutput",
394-
- "path": "path/to/file",
395-
+ "path": "path/to/file2",
396-
"checksum": {
397-
- "value": "45372qe",
398-
+ "value": "42472qet",
399-
"algorithm": "nextflow",
400-
"mode": "standard"
401-
},
402-
- "source": "lid://123987/file.bam",
403-
+ "source": "lid://123987/file2.bam",
404-
"workflowRun": "lid://123987/",
405-
"taskRun": null,
406-
- "size": 1234,
407-
+ "size": 1235,
408-
"createdAt": "1970-01-02T10:17:36.789Z",
409-
"modifiedAt": "1970-01-02T10:17:36.789Z",
410-
"labels": null
393+
"spec": {
394+
- "path": "path/to/file",
395+
+ "path": "path/to/file2",
396+
"checksum": {
397+
- "value": "45372qe",
398+
+ "value": "42472qet",
399+
"algorithm": "nextflow",
400+
"mode": "standard"
401+
},
402+
- "source": "lid://123987/file.bam",
403+
+ "source": "lid://123987/file2.bam",
404+
"workflowRun": "lid://123987/",
405+
"taskRun": null,
406+
- "size": 1234,
407+
+ "size": 1235,
408+
"createdAt": "1970-01-02T10:17:36.789Z",
409+
"modifiedAt": "1970-01-02T10:17:36.789Z",
410+
"labels": null
411411
'''
412412

413413
when:

‎modules/nf-lineage/src/test/nextflow/lineage/fs/LinFileSystemProviderTest.groovy

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@ class LinFileSystemProviderTest extends Specification {
123123
def output = data.resolve("output.txt")
124124
output.text = "Hello, World!"
125125
outputMeta.mkdirs()
126-
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","path":"'+output.toString()+'"}'
126+
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
127127

128128
Global.session = Mock(Session) { getConfig()>>config }
129129
and:
@@ -179,7 +179,7 @@ class LinFileSystemProviderTest extends Specification {
179179
def config = [lineage:[store:[location:wdir.toString()]]]
180180
def outputMeta = wdir.resolve("12345")
181181
outputMeta.mkdirs()
182-
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"WorkflowRun","sessionId":"session","name":"run_name","params":[{"type":"String","name":"param1","value":"value1"}]}'
182+
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"WorkflowRun","spec":{"sessionId":"session","name":"run_name","params":[{"type":"String","name":"param1","value":"value1"}]}}'
183183

184184
Global.session = Mock(Session) { getConfig()>>config }
185185
and:
@@ -238,7 +238,7 @@ class LinFileSystemProviderTest extends Specification {
238238
def output = data.resolve("output.txt")
239239
output.text = "Hello, World!"
240240
outputMeta.mkdirs()
241-
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","path":"'+output.toString()+'"}'
241+
outputMeta.resolve(".data.json").text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"'+output.toString()+'"}}'
242242

243243
Global.session = Mock(Session) { getConfig()>>config }
244244
and:
@@ -278,8 +278,8 @@ class LinFileSystemProviderTest extends Specification {
278278
output1.resolve('file3.txt').text = 'file3'
279279
wdir.resolve('12345/output1').mkdirs()
280280
wdir.resolve('12345/output2').mkdirs()
281-
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun"}'
282-
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput", "path": "' + output1.toString() + '"}'
281+
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"dummy"}}'
282+
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output1.toString() + '"}}'
283283

284284
and:
285285
def config = [lineage:[store:[location:wdir.toString()]]]
@@ -403,7 +403,7 @@ class LinFileSystemProviderTest extends Specification {
403403
output.resolve('abc').text = 'file1'
404404
output.resolve('.foo').text = 'file2'
405405
wdir.resolve('12345/output').mkdirs()
406-
wdir.resolve('12345/output/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput", "path": "' + output.toString() + '"}'
406+
wdir.resolve('12345/output/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + output.toString() + '"}}'
407407
and:
408408
def provider = new LinFileSystemProvider()
409409
def lid1 = provider.getPath(LinPath.asUri('lid://12345/output/abc'))
@@ -423,7 +423,7 @@ class LinFileSystemProviderTest extends Specification {
423423
def file = data.resolve('abc')
424424
file.text = 'Hello'
425425
wdir.resolve('12345/abc').mkdirs()
426-
wdir.resolve('12345/abc/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput", "path":"' + file.toString() + '"}'
426+
wdir.resolve('12345/abc/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path":"' + file.toString() + '"}}'
427427
and:
428428
Global.session = Mock(Session) { getConfig()>>config }
429429
and:

‎modules/nf-lineage/src/test/nextflow/lineage/fs/LinPathTest.groovy

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -161,9 +161,9 @@ class LinPathTest extends Specification {
161161

162162
wdir.resolve('12345/output1').mkdirs()
163163
wdir.resolve('12345/path/to/file2.txt').mkdirs()
164-
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun"}'
165-
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput", "path": "' + outputFolder.toString() + '"}'
166-
wdir.resolve('12345/path/to/file2.txt/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput", "path": "' + outputFile.toString() + '"}'
164+
wdir.resolve('12345/.data.json').text = '{"version":"lineage/v1beta1","kind":"TaskRun","spec":{"name":"test"}}'
165+
wdir.resolve('12345/output1/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFolder.toString() + '"}}'
166+
wdir.resolve('12345/path/to/file2.txt/.data.json').text = '{"version":"lineage/v1beta1","kind":"FileOutput","spec":{"path": "' + outputFile.toString() + '"}}'
167167
def time = OffsetDateTime.now()
168168
def wfResultsMetadata = new LinEncoder().withPrettyPrint(true).encode(new WorkflowOutput(time, "lid://1234", [new Parameter( "Path", "a", "lid://1234/a.txt")]))
169169
wdir.resolve('5678/').mkdirs()

‎modules/nf-lineage/src/test/nextflow/lineage/serde/LinEncoderTest.groovy

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,7 @@ class LinEncoderTest extends Specification{
167167
def encoded = encoder.encode(wfResults)
168168
def object = encoder.decode(encoded)
169169
then:
170-
encoded == '{"version":"lineage/v1beta1","kind":"WorkflowOutput","createdAt":null,"workflowRun":"lid://1234","output":null}'
170+
encoded == '{"version":"lineage/v1beta1","kind":"WorkflowOutput","spec":{"createdAt":null,"workflowRun":"lid://1234","output":null}}'
171171
def result = object as WorkflowOutput
172172
result.createdAt == null
173173

0 commit comments

Comments
 (0)