Skip to content

Commit e072820

Browse files
rjrudinrjrudin
authored
Merge pull request #1772 from marklogic/feature/21471-tools-tests
MLE-21471 Getting ml-development-tools tests running on 12 again
2 parents e231f73 + 56269ee commit e072820

File tree

7 files changed

+144
-90
lines changed

7 files changed

+144
-90
lines changed

‎test-app/.env renamed to ‎.env

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ MARKLOGIC_IMAGE=progressofficial/marklogic-db:latest
44
MARKLOGIC_LOGS_VOLUME=./docker/marklogic/logs
55

66
# This image should be used instead of the above image when testing functions that only work with MarkLogic 12.
7-
# MARKLOGIC_IMAGE=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12
7+
#MARKLOGIC_IMAGE=ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-12

‎CONTRIBUTING.md

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,7 @@ be a local instance or it may be running in a Docker container. If you would lik
3333
instance, you may create the container with the following commands (starting in the project root directory):
3434

3535
```
36-
cd test-app
3736
docker-compose up -d --build
38-
cd ..
3937
```
4038

4139
Once you have a MarkLogic instance ready, the application is then deployed via the following command:

‎Jenkinsfile

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,18 +47,14 @@ def runTests(String image) {
4747
./gradlew marklogic-client-api:test || true
4848
'''
4949

50-
// Ignoring these for now on MarkLogic 12 as they are failing due to the generated .mjs modules not including
51-
// "export default" in the last line, which is now required by the MarkLogic 12 nightly build as of 2025-05-05.
52-
if (!image.endsWith("12")) {
53-
sh label:'run ml-development-tools tests', script: '''#!/bin/bash
54-
export JAVA_HOME=$JAVA_HOME_DIR
55-
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
56-
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
57-
cd java-client-api
58-
mkdir -p ml-development-tools/build/test-results/test
59-
./gradlew ml-development-tools:test || true
60-
'''
61-
}
50+
sh label:'run ml-development-tools tests', script: '''#!/bin/bash
51+
export JAVA_HOME=$JAVA_HOME_DIR
52+
export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
53+
export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
54+
cd java-client-api
55+
mkdir -p ml-development-tools/build/test-results/test
56+
./gradlew ml-development-tools:test || true
57+
'''
6258

6359
sh label:'run fragile functional tests', script: '''#!/bin/bash
6460
export JAVA_HOME=$JAVA_HOME_DIR

‎test-app/docker-compose.yaml renamed to ‎docker-compose.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: docker-tests-marklogic-javaclient-test-app
1+
name: docker-tests-java-client
22

33
services:
44

‎ml-development-tools/build.gradle

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,11 @@ dependencies {
2222
testCompileOnly gradleTestKit()
2323

2424
testImplementation 'com.squareup.okhttp3:okhttp:4.12.0'
25+
26+
// Needed for MarkLogicVersion class
27+
testImplementation ("com.marklogic:marklogic-junit5:1.5.0") {
28+
exclude module: "marklogic-client-api"
29+
}
2530
}
2631

2732
// Added to avoid problem where processResources fails because - somehow - the plugin properties file is getting
@@ -30,7 +35,8 @@ tasks.processResources {
3035
duplicatesStrategy = "exclude"
3136
}
3237

33-
task mlDevelopmentToolsJar(type: Jar, dependsOn: classes) {
38+
tasks.register("mlDevelopmentToolsJar", Jar) {
39+
dependsOn classes
3440
archivesBaseName = 'ml-development-tools'
3541
}
3642

@@ -74,11 +80,17 @@ compileTestKotlin {
7480
kotlinOptions.jvmTarget = '1.8'
7581
}
7682

77-
task generateTests(type: JavaExec) {
83+
tasks.register("generateTests", JavaExec) {
7884
classpath = sourceSets.test.runtimeClasspath
79-
main = 'com.marklogic.client.test.dbfunction.FntestgenKt'
85+
mainClass = 'com.marklogic.client.test.dbfunction.FntestgenKt'
8086
args = ['./src/test/', 'latest']
8187
}
8288

89+
tasks.register("fixMjsModulesForMarkLogic12", JavaExec) {
90+
classpath = sourceSets.test.runtimeClasspath
91+
mainClass = 'com.marklogic.client.test.FixMjsModulesForMarkLogic12'
92+
}
93+
8394
// Allows running "./gradlew test" without having to remember to generate the tests first.
84-
test.dependsOn generateTests
95+
test.dependsOn generateTests, fixMjsModulesForMarkLogic12
96+
fixMjsModulesForMarkLogic12.mustRunAfter generateTests

‎ml-development-tools/src/test/java/com/marklogic/client/test/FixMjsModulesForMarkLogic12.java

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
/*
2+
* Copyright © 2025 MarkLogic Corporation. All Rights Reserved.
3+
*/
4+
package com.marklogic.client.test;
5+
6+
import com.marklogic.client.DatabaseClient;
7+
import com.marklogic.client.datamovement.DataMovementManager;
8+
import com.marklogic.client.datamovement.QueryBatcher;
9+
import com.marklogic.client.document.DocumentWriteSet;
10+
import com.marklogic.client.document.TextDocumentManager;
11+
import com.marklogic.client.io.DocumentMetadataHandle;
12+
import com.marklogic.client.io.Format;
13+
import com.marklogic.client.io.StringHandle;
14+
import com.marklogic.client.test.dbfunction.DBFunctionTestUtil;
15+
import com.marklogic.junit5.MarkLogicVersion;
16+
import com.networknt.schema.utils.StringUtils;
17+
18+
/**
19+
* Program for "fixing" MJS modules when running against MarkLogic 12. A nightly version of 12 from early May
20+
* 2025 requires that MJS modules use "export default" in the last line of the module to return data.
21+
* <p>
22+
* This approach is being used as it's easier to maintain than trying to modify the Kotlin-based code generator for
23+
* MJS modules.
24+
*/
25+
public class FixMjsModulesForMarkLogic12 {
26+
27+
private static final DocumentMetadataHandle MODULE_METADATA = new DocumentMetadataHandle()
28+
.withPermission("rest-reader", DocumentMetadataHandle.Capability.EXECUTE)
29+
.withPermission("rest-writer", DocumentMetadataHandle.Capability.UPDATE);
30+
31+
public static void main(String[] args) {
32+
try (DatabaseClient client = DBFunctionTestUtil.makeAdminClient("java-unittest-modules")) {
33+
String version = client.newServerEval().javascript("xdmp.version()").evalAs(String.class);
34+
if (new MarkLogicVersion(version).getMajor() < 12) {
35+
return;
36+
}
37+
38+
DataMovementManager dataMovementManager = client.newDataMovementManager();
39+
final TextDocumentManager documentManager = client.newTextDocumentManager();
40+
final DocumentWriteSet modulesToUpdate = documentManager.newWriteSet();
41+
42+
QueryBatcher queryBatcher = dataMovementManager.newQueryBatcher(
43+
client.newQueryManager().newStructuredQueryBuilder().trueQuery()
44+
)
45+
.withThreadCount(4)
46+
.withBatchSize(100)
47+
.onUrisReady(batch -> {
48+
for (String uri : batch.getItems()) {
49+
if (!uri.startsWith("/dbf/") || !uri.endsWith(".mjs")) {
50+
continue;
51+
}
52+
String content = documentManager.read(uri, new StringHandle()).get();
53+
String[] lines = getModuleContentAsLines(content);
54+
if (!lines[lines.length - 1].startsWith("export default")) {
55+
System.out.println("Fixing: " + uri);
56+
String newContent = fixModuleContent(lines);
57+
modulesToUpdate.add(uri, MODULE_METADATA, new StringHandle(newContent).withFormat(Format.TEXT));
58+
} else {
59+
System.out.println("Not fixing: " + uri);
60+
}
61+
}
62+
});
63+
64+
dataMovementManager.startJob(queryBatcher);
65+
queryBatcher.awaitCompletion();
66+
dataMovementManager.stopJob(queryBatcher);
67+
68+
System.out.println("Count of modules to update: " + modulesToUpdate.size());
69+
if (modulesToUpdate.size() > 0) {
70+
documentManager.write(modulesToUpdate);
71+
}
72+
73+
System.out.println("Done!");
74+
}
75+
}
76+
77+
private static String[] getModuleContentAsLines(String content) {
78+
String[] lines = content.split("\n");
79+
// Drop the last line if it's blank, which oddly happens in a small handful of the generated modules.
80+
if (StringUtils.isBlank(lines[lines.length - 1])) {
81+
String[] newLines = new String[lines.length - 1];
82+
System.arraycopy(lines, 0, newLines, 0, lines.length - 1);
83+
return newLines;
84+
}
85+
return lines;
86+
}
87+
88+
private static String fixModuleContent(String[] lines) {
89+
StringBuilder newContent = new StringBuilder();
90+
for (int i = 0; i < lines.length - 2; i++) {
91+
newContent.append(lines[i]).append("\n");
92+
}
93+
newContent.append("const theOutputToExport = ").append(lines[lines.length - 1]).append("\n");
94+
newContent.append("export default theOutputToExport");
95+
return newContent.toString();
96+
}
97+
}

‎ml-development-tools/src/test/java/com/marklogic/client/test/dbfunction/DBFunctionTestUtil.java

Lines changed: 21 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -2,82 +2,33 @@
22

33
import com.marklogic.client.DatabaseClient;
44
import com.marklogic.client.DatabaseClientFactory;
5-
import okhttp3.HttpUrl;
6-
import okhttp3.OkHttpClient;
7-
import okhttp3.Request;
8-
import okhttp3.Response;
9-
10-
import java.io.IOException;
11-
import java.io.InputStream;
12-
import java.net.URL;
135

146
public class DBFunctionTestUtil {
15-
// used for test endpoints that inspect request and generate response
16-
public final static DatabaseClient db = makeTestClient();
177

18-
// used for bulk test endpoints that depend on DMSDK
19-
public final static DatabaseClient restDb = makeRestClient();
8+
// used for test endpoints that inspect request and generate response
9+
public final static DatabaseClient db = makeTestClientImpl(
10+
new DatabaseClientFactory.DigestAuthContext("rest-reader", "x")
11+
);
2012

21-
// used for test endpoints that need an elevated privilege
22-
public final static DatabaseClient adminDb = makeAdminTestClient();
13+
// used for test endpoints that need an elevated privilege
14+
public final static DatabaseClient adminDb = makeTestClientImpl(
15+
new DatabaseClientFactory.DigestAuthContext("admin", "admin")
16+
);
2317

24-
private static DatabaseClient makeRestClient() {
25-
return makeRestClientImpl(
26-
new DatabaseClientFactory.DigestAuthContext("rest-reader", "x")
27-
);
28-
}
29-
private static DatabaseClient makeTestClient() {
30-
return makeTestClientImpl(
31-
new DatabaseClientFactory.DigestAuthContext("rest-reader", "x")
32-
);
33-
}
34-
private static DatabaseClient makeAdminTestClient() {
35-
return makeTestClientImpl(
36-
new DatabaseClientFactory.DigestAuthContext("admin", "admin")
37-
);
38-
}
39-
private static DatabaseClient makeRestClientImpl(DatabaseClientFactory.DigestAuthContext auth) {
40-
return makeClientImpl(auth, "8012", false);
41-
}
42-
private static DatabaseClient makeTestClientImpl(DatabaseClientFactory.DigestAuthContext auth) {
43-
return makeClientImpl(auth, "8013", true);
44-
}
45-
private static DatabaseClient makeClientImpl(
46-
DatabaseClientFactory.DigestAuthContext auth, String defaultPort, boolean withCheck
47-
) {
48-
String host = System.getProperty("TEST_HOST", "localhost");
49-
int port = Integer.parseInt(System.getProperty("TEST_PORT", defaultPort));
18+
public static DatabaseClient makeAdminClient(String database) {
19+
return makeClientImpl(new DatabaseClientFactory.DigestAuthContext("admin", "admin"), 8000, database);
20+
}
5021

51-
DatabaseClient db = DatabaseClientFactory.newClient(host, port, auth);
22+
private static DatabaseClient makeTestClientImpl(DatabaseClientFactory.DigestAuthContext auth) {
23+
return makeClientImpl(auth, 8013, null);
24+
}
5225

53-
if (withCheck) {
54-
try {
55-
OkHttpClient client = (OkHttpClient) db.getClientImplementation();
56-
// TODO: better alternative to ping for non-REST server
57-
Response response = client.newCall(new Request.Builder().url(
58-
new HttpUrl.Builder()
59-
.scheme("http")
60-
.host(host)
61-
.port(port)
62-
.encodedPath("/")
63-
.build()
64-
).build()
65-
).execute();
66-
int statusCode = response.code();
67-
if (statusCode >= 300 && statusCode != 404) {
68-
throw new RuntimeException(statusCode+" "+response.message());
69-
}
70-
} catch (IOException e) {
71-
throw new RuntimeException(e);
72-
}
73-
}
26+
private static DatabaseClient makeClientImpl(DatabaseClientFactory.SecurityContext auth, int defaultPort, String database) {
27+
String host = System.getProperty("TEST_HOST", "localhost");
28+
int port = Integer.parseInt(System.getProperty("TEST_PORT", Integer.toString(defaultPort)));
7429

75-
return db;
76-
}
77-
public static URL getResource(String name) {
78-
return DBFunctionTestUtil.class.getClassLoader().getResource(name);
79-
}
80-
public static InputStream getResourceAsStream(String name) {
81-
return DBFunctionTestUtil.class.getClassLoader().getResourceAsStream(name);
82-
}
30+
return database != null ?
31+
DatabaseClientFactory.newClient(host, port, database, auth) :
32+
DatabaseClientFactory.newClient(host, port, auth);
33+
}
8334
}

0 commit comments

Comments
 (0)