Skip to content

Commit 0e91533

Browse files
akolarkunnuakolarkunnu
authored
Merge branch 'main' into prepostfuncvalidation
2 parents bb0549e + 532284e commit 0e91533

File tree

434 files changed

+17509
-1560
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

434 files changed

+17509
-1560
lines changed

‎.github/workflows/CI-workflow.yml

Lines changed: 12 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -84,18 +84,23 @@ jobs:
8484
echo "::add-mask::$COHERE_KEY" &&
8585
echo "build and run tests" && ./gradlew build -x spotlessJava &&
8686
echo "Publish to Maven Local" && ./gradlew publishToMavenLocal -x spotlessJava &&
87-
echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3 -x spotlessJava
88-
echo "Run Jacoco test coverage" && && ./gradlew jacocoTestReport && cp -v plugin/build/reports/jacoco/test/jacocoTestReport.xml ./jacocoTestReport.xml'
87+
echo "Multi Nodes Integration Testing" && ./gradlew integTest -PnumNodes=3 -x spotlessJava'
8988
plugin=`basename $(ls plugin/build/distributions/*.zip)`
9089
echo $plugin
9190
mv -v plugin/build/distributions/$plugin ./
9291
echo "build-test-linux=$plugin" >> $GITHUB_OUTPUT
9392
93+
- name: Upload Coverage Report
94+
uses: codecov/codecov-action@v3
95+
with:
96+
flags: ml-commons
97+
token: ${{ secrets.CODECOV_TOKEN }}
98+
9499
- uses: actions/upload-artifact@v4
95-
if: ${{ matrix.os }} == "ubuntu-latest"
96100
with:
97-
name: coverage-report-${{ matrix.os }}-${{ matrix.java }}
98-
path: ./jacocoTestReport.xml
101+
name: ml-plugin-linux-${{ matrix.java }}
102+
path: ${{ steps.step-build-test-linux.outputs.build-test-linux }}
103+
if-no-files-found: error
99104

100105

101106
Test-ml-linux-docker:
@@ -164,8 +169,8 @@ jobs:
164169
- name: Generate Password For Admin
165170
id: genpass
166171
run: |
167-
PASSWORD=$(openssl rand -base64 20 | tr -dc 'A-Za-z0-9!@#$%^&*()_+=-')
168-
echo "password={$PASSWORD}" >> $GITHUB_OUTPUT
172+
PASSWORD=$(openssl rand -base64 20 | tr -dc 'A-Za-z0-9!@#$%^&*()_+=-')
173+
echo "password={$PASSWORD}" >> $GITHUB_OUTPUT
169174
- name: Run Docker Image
170175
if: env.imagePresent == 'true'
171176
run: |
@@ -195,24 +200,6 @@ jobs:
195200
flags: ml-commons
196201
token: ${{ secrets.CODECOV_TOKEN }}
197202

198-
Precommit-codecov:
199-
needs: Build-ml-linux
200-
strategy:
201-
matrix:
202-
java: [21, 23]
203-
os: [ubuntu-latest]
204-
runs-on: ${{ matrix.os }}
205-
steps:
206-
- uses: actions/download-artifact@v4
207-
with:
208-
name: coverage-report-${{ matrix.os }}-${{ matrix.java }}
209-
path: ./
210-
- name: Upload Coverage Report
211-
uses: codecov/codecov-action@v5
212-
with:
213-
token: ${{ secrets.CODECOV_TOKEN }}
214-
files: ./jacocoTestReport.xml
215-
216203
Build-ml-windows:
217204
strategy:
218205
matrix:

‎build.gradle

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@ buildscript {
1111
ext {
1212
opensearch_group = "org.opensearch"
1313
isSnapshot = "true" == System.getProperty("build.snapshot", "true")
14-
opensearch_version = System.getProperty("opensearch.version", "3.0.0-alpha1-SNAPSHOT")
15-
buildVersionQualifier = System.getProperty("build.version_qualifier", "alpha1")
14+
opensearch_version = System.getProperty("opensearch.version", "3.1.0-SNAPSHOT")
15+
buildVersionQualifier = System.getProperty("build.version_qualifier", "")
1616
asm_version = "9.7"
1717

1818
// 2.0.0-rc1-SNAPSHOT -> 2.0.0.0-rc1-SNAPSHOT

‎common/src/main/java/org/opensearch/ml/common/CommonValue.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ public class CommonValue {
4545
public static final String ML_MEMORY_MESSAGE_INDEX = ".plugins-ml-memory-message";
4646
public static final String ML_STOP_WORDS_INDEX = ".plugins-ml-stop-words";
4747
public static final String TASK_POLLING_JOB_INDEX = ".ml_commons_task_polling_job";
48+
public static final String MCP_SESSION_MANAGEMENT_INDEX = ".plugins-ml-mcp-session-management";
4849
public static final Set<String> stopWordsIndices = ImmutableSet.of(".plugins-ml-stop-words");
4950
public static final String TOOL_PARAMETERS_PREFIX = "tools.parameters.";
5051

@@ -58,6 +59,7 @@ public class CommonValue {
5859
public static final String ML_AGENT_INDEX_MAPPING_PATH = "index-mappings/ml_agent.json";
5960
public static final String ML_MEMORY_META_INDEX_MAPPING_PATH = "index-mappings/ml_memory_meta.json";
6061
public static final String ML_MEMORY_MESSAGE_INDEX_MAPPING_PATH = "index-mappings/ml_memory_message.json";
62+
public static final String ML_MCP_SESSION_MANAGEMENT_INDEX_MAPPING_PATH = "index-mappings/ml_mcp_session_management.json";
6163

6264
// Calculate Versions independently of OpenSearch core version
6365
public static final Version VERSION_2_11_0 = Version.fromString("2.11.0");
@@ -69,4 +71,34 @@ public class CommonValue {
6971
public static final Version VERSION_2_17_0 = Version.fromString("2.17.0");
7072
public static final Version VERSION_2_18_0 = Version.fromString("2.18.0");
7173
public static final Version VERSION_2_19_0 = Version.fromString("2.19.0");
74+
public static final Version VERSION_3_0_0 = Version.fromString("3.0.0");
75+
public static final Version VERSION_3_1_0 = Version.fromString("3.1.0");
76+
77+
// Connector Constants
78+
public static final String NAME_FIELD = "name";
79+
public static final String VERSION_FIELD = "version";
80+
public static final String DESCRIPTION_FIELD = "description";
81+
public static final String PROTOCOL_FIELD = "protocol";
82+
public static final String CREDENTIAL_FIELD = "credential";
83+
public static final String PARAMETERS_FIELD = "parameters";
84+
public static final String CREATED_TIME_FIELD = "created_time";
85+
public static final String LAST_UPDATED_TIME_FIELD = "last_updated_time";
86+
public static final String BACKEND_ROLES_FIELD = "backend_roles";
87+
public static final String OWNER_FIELD = "owner";
88+
public static final String ACCESS_FIELD = "access";
89+
public static final String CLIENT_CONFIG_FIELD = "client_config";
90+
public static final String URL_FIELD = "url";
91+
public static final String HEADERS_FIELD = "headers";
92+
93+
// MCP Constants
94+
public static final String MCP_TOOL_NAME_FIELD = "name";
95+
public static final String MCP_TOOL_DESCRIPTION_FIELD = "description";
96+
public static final String MCP_TOOL_INPUT_SCHEMA_FIELD = "inputSchema";
97+
public static final String MCP_SYNC_CLIENT = "mcp_sync_client";
98+
public static final String MCP_TOOLS_FIELD = "tools";
99+
public static final String MCP_CONNECTORS_FIELD = "mcp_connectors";
100+
public static final String MCP_CONNECTOR_ID_FIELD = "mcp_connector_id";
101+
102+
// TOOL Constants
103+
public static final String TOOL_INPUT_SCHEMA_FIELD = "input_schema";
72104
}

‎common/src/main/java/org/opensearch/ml/common/MLAgentType.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
public enum MLAgentType {
1111
FLOW,
1212
CONVERSATIONAL,
13-
CONVERSATIONAL_FLOW;
13+
CONVERSATIONAL_FLOW,
14+
PLAN_EXECUTE_AND_REFLECT;
1415

1516
public static MLAgentType from(String value) {
1617
if (value == null) {

‎common/src/main/java/org/opensearch/ml/common/MLIndex.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@
1313
import static org.opensearch.ml.common.CommonValue.ML_CONNECTOR_INDEX_MAPPING_PATH;
1414
import static org.opensearch.ml.common.CommonValue.ML_CONTROLLER_INDEX;
1515
import static org.opensearch.ml.common.CommonValue.ML_CONTROLLER_INDEX_MAPPING_PATH;
16+
import static org.opensearch.ml.common.CommonValue.ML_MCP_SESSION_MANAGEMENT_INDEX_MAPPING_PATH;
1617
import static org.opensearch.ml.common.CommonValue.ML_MEMORY_MESSAGE_INDEX;
1718
import static org.opensearch.ml.common.CommonValue.ML_MEMORY_MESSAGE_INDEX_MAPPING_PATH;
1819
import static org.opensearch.ml.common.CommonValue.ML_MEMORY_META_INDEX;
@@ -38,7 +39,8 @@ public enum MLIndex {
3839
CONTROLLER(ML_CONTROLLER_INDEX, false, ML_CONTROLLER_INDEX_MAPPING_PATH),
3940
AGENT(ML_AGENT_INDEX, false, ML_AGENT_INDEX_MAPPING_PATH),
4041
MEMORY_META(ML_MEMORY_META_INDEX, false, ML_MEMORY_META_INDEX_MAPPING_PATH),
41-
MEMORY_MESSAGE(ML_MEMORY_MESSAGE_INDEX, false, ML_MEMORY_MESSAGE_INDEX_MAPPING_PATH);
42+
MEMORY_MESSAGE(ML_MEMORY_MESSAGE_INDEX, false, ML_MEMORY_MESSAGE_INDEX_MAPPING_PATH),
43+
MCP_SESSION_MANAGEMENT(ML_MCP_SESSION_MANAGEMENT_INDEX_MAPPING_PATH, false, ML_MCP_SESSION_MANAGEMENT_INDEX_MAPPING_PATH);
4244

4345
private final String indexName;
4446
// whether we use an alias for the index

‎common/src/main/java/org/opensearch/ml/common/MLModel.java

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,13 +32,15 @@
3232
import org.opensearch.core.xcontent.XContentParser;
3333
import org.opensearch.ml.common.connector.Connector;
3434
import org.opensearch.ml.common.controller.MLRateLimiter;
35+
import org.opensearch.ml.common.model.BaseModelConfig;
3536
import org.opensearch.ml.common.model.Guardrails;
3637
import org.opensearch.ml.common.model.MLDeploySetting;
3738
import org.opensearch.ml.common.model.MLModelConfig;
3839
import org.opensearch.ml.common.model.MLModelFormat;
3940
import org.opensearch.ml.common.model.MLModelState;
4041
import org.opensearch.ml.common.model.MetricsCorrelationModelConfig;
4142
import org.opensearch.ml.common.model.QuestionAnsweringModelConfig;
43+
import org.opensearch.ml.common.model.RemoteModelConfig;
4244
import org.opensearch.ml.common.model.TextEmbeddingModelConfig;
4345

4446
import lombok.Builder;
@@ -278,8 +280,12 @@ public MLModel(StreamInput input) throws IOException {
278280
modelConfig = new MetricsCorrelationModelConfig(input);
279281
} else if (algorithm.equals(FunctionName.QUESTION_ANSWERING)) {
280282
modelConfig = new QuestionAnsweringModelConfig(input);
281-
} else {
283+
} else if (algorithm.equals(FunctionName.TEXT_EMBEDDING)) {
282284
modelConfig = new TextEmbeddingModelConfig(input);
285+
} else if (algorithm.equals(FunctionName.REMOTE)) {
286+
modelConfig = new RemoteModelConfig(input);
287+
} else {
288+
modelConfig = new BaseModelConfig(input);
283289
}
284290
}
285291
if (input.readBoolean()) {
@@ -623,8 +629,12 @@ public static MLModel parse(XContentParser parser, String algorithmName) throws
623629
modelConfig = MetricsCorrelationModelConfig.parse(parser);
624630
} else if (FunctionName.QUESTION_ANSWERING.name().equals(algorithmName)) {
625631
modelConfig = QuestionAnsweringModelConfig.parse(parser);
626-
} else {
632+
} else if (FunctionName.TEXT_EMBEDDING.name().equals(algorithmName)) {
627633
modelConfig = TextEmbeddingModelConfig.parse(parser);
634+
} else if (FunctionName.REMOTE.name().equals(algorithmName)) {
635+
modelConfig = RemoteModelConfig.parse(parser);
636+
} else {
637+
modelConfig = BaseModelConfig.parse(parser);
628638
}
629639
break;
630640
case DEPLOY_SETTING_FIELD:

‎common/src/main/java/org/opensearch/ml/common/MLTask.java

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,9 @@ public class MLTask implements ToXContentObject, Writeable {
5050
public static final String ERROR_FIELD = "error";
5151
public static final String IS_ASYNC_TASK_FIELD = "is_async";
5252
public static final String REMOTE_JOB_FIELD = "remote_job";
53+
public static final String RESPONSE_FIELD = "response";
5354
public static final Version MINIMAL_SUPPORTED_VERSION_FOR_BATCH_PREDICTION_JOB = CommonValue.VERSION_2_17_0;
55+
public static final Version MINIMAL_SUPPORTED_VERSION_FOR_RESPONSE_FIELD = CommonValue.VERSION_3_0_0;
5456

5557
@Setter
5658
private String taskId;
@@ -74,6 +76,8 @@ public class MLTask implements ToXContentObject, Writeable {
7476
private boolean async;
7577
@Setter
7678
private Map<String, Object> remoteJob;
79+
@Setter
80+
private Map<String, Object> response;
7781
private String tenantId;
7882

7983
@Builder(toBuilder = true)
@@ -93,6 +97,7 @@ public MLTask(
9397
User user,
9498
boolean async,
9599
Map<String, Object> remoteJob,
100+
Map<String, Object> response,
96101
String tenantId
97102
) {
98103
this.taskId = taskId;
@@ -110,6 +115,7 @@ public MLTask(
110115
this.user = user;
111116
this.async = async;
112117
this.remoteJob = remoteJob;
118+
this.response = response;
113119
this.tenantId = tenantId;
114120
}
115121

@@ -142,6 +148,11 @@ public MLTask(StreamInput input) throws IOException {
142148
this.remoteJob = input.readMap(StreamInput::readString, StreamInput::readGenericValue);
143149
}
144150
}
151+
if (streamInputVersion.onOrAfter(MLTask.MINIMAL_SUPPORTED_VERSION_FOR_RESPONSE_FIELD)) {
152+
if (input.readBoolean()) {
153+
this.response = input.readMap(StreamInput::readString, StreamInput::readGenericValue);
154+
}
155+
}
145156
tenantId = streamInputVersion.onOrAfter(VERSION_2_19_0) ? input.readOptionalString() : null;
146157
}
147158

@@ -179,6 +190,14 @@ public void writeTo(StreamOutput out) throws IOException {
179190
out.writeBoolean(false);
180191
}
181192
}
193+
if (streamOutputVersion.onOrAfter(MLTask.MINIMAL_SUPPORTED_VERSION_FOR_RESPONSE_FIELD)) {
194+
if (response != null) {
195+
out.writeBoolean(true);
196+
out.writeMap(response, StreamOutput::writeString, StreamOutput::writeGenericValue);
197+
} else {
198+
out.writeBoolean(false);
199+
}
200+
}
182201
if (streamOutputVersion.onOrAfter(VERSION_2_19_0)) {
183202
out.writeOptionalString(tenantId);
184203
}
@@ -230,6 +249,9 @@ public XContentBuilder toXContent(XContentBuilder xContentBuilder, Params params
230249
if (remoteJob != null) {
231250
builder.field(REMOTE_JOB_FIELD, remoteJob);
232251
}
252+
if (response != null) {
253+
builder.field(RESPONSE_FIELD, response);
254+
}
233255
if (tenantId != null) {
234256
builder.field(TENANT_ID_FIELD, tenantId);
235257
}
@@ -256,6 +278,7 @@ public static MLTask parse(XContentParser parser) throws IOException {
256278
User user = null;
257279
boolean async = false;
258280
Map<String, Object> remoteJob = null;
281+
Map<String, Object> response = null;
259282
String tenantId = null;
260283

261284
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
@@ -317,6 +340,9 @@ public static MLTask parse(XContentParser parser) throws IOException {
317340
case REMOTE_JOB_FIELD:
318341
remoteJob = parser.map();
319342
break;
343+
case RESPONSE_FIELD:
344+
response = parser.map();
345+
break;
320346
case TENANT_ID_FIELD:
321347
tenantId = parser.textOrNull();
322348
break;
@@ -342,6 +368,7 @@ public static MLTask parse(XContentParser parser) throws IOException {
342368
.user(user)
343369
.async(async)
344370
.remoteJob(remoteJob)
371+
.response(response)
345372
.tenantId(tenantId)
346373
.build();
347374
}

‎common/src/main/java/org/opensearch/ml/common/MLTaskType.java

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,6 @@ public enum MLTaskType {
1717
REGISTER_MODEL,
1818
DEPLOY_MODEL,
1919
BATCH_INGEST,
20-
BATCH_PREDICTION
20+
BATCH_PREDICTION,
21+
AGENT_EXECUTION
2122
}

0 commit comments

Comments
 (0)