From 1bf250b4b22da93c3399604b79dca6c8594f5a1b Mon Sep 17 00:00:00 2001 From: Yann Dirson Date: Tue, 5 Mar 2024 16:55:30 +0100 Subject: [PATCH 001/149] Quicktest_vm_lifecycle: use requested SR not default SR This test did not exercised the SR type expected by user, and notably failed when no default SR was defined. Adds the option of installing a VM in a non-default SR using VM.with_new. Signed-off-by: Yann Dirson --- ocaml/quicktest/qt.ml | 26 ++++++++++++++--------- ocaml/quicktest/qt.mli | 7 +++++- ocaml/quicktest/quicktest_vm_lifecycle.ml | 12 ++++++++--- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/ocaml/quicktest/qt.ml b/ocaml/quicktest/qt.ml index 1764f12ce8f..d390f0dfc38 100644 --- a/ocaml/quicktest/qt.ml +++ b/ocaml/quicktest/qt.ml @@ -132,26 +132,32 @@ module VM = struct Some x end - let install rpc session_id ~template ~name = + let install rpc session_id ~template ~name ?sr () = let template_uuid = Client.Client.VM.get_uuid ~rpc ~session_id ~self:template in - let newvm_uuid = - cli_cmd - [ - "vm-install" - ; "template-uuid=" ^ template_uuid - ; "new-name-label=" ^ name - ] + let cmd = + ["vm-install"; "template-uuid=" ^ template_uuid; "new-name-label=" ^ name] in + let sr_uuid = + Option.map + (fun sr -> Client.Client.SR.get_uuid ~rpc ~session_id ~self:sr) + sr + in + let cmd = + cmd @ Option.fold ~none:[] ~some:(fun x -> ["sr-uuid=" ^ x]) sr_uuid + in + let newvm_uuid = cli_cmd cmd in Client.Client.VM.get_by_uuid ~rpc ~session_id ~uuid:newvm_uuid let uninstall rpc session_id vm = let uuid = Client.Client.VM.get_uuid ~rpc ~session_id ~self:vm in cli_cmd ["vm-uninstall"; "uuid=" ^ uuid; "--force"] |> ignore - let with_new rpc session_id ~template f = - let vm = install rpc session_id ~template ~name:"temp_quicktest_vm" in + let with_new rpc session_id ~template ?sr f = + let vm = + install rpc session_id ~template ~name:"temp_quicktest_vm" ?sr () + in Xapi_stdext_pervasives.Pervasiveext.finally (fun () -> f vm) (fun () -> uninstall rpc session_id vm) diff --git a/ocaml/quicktest/qt.mli b/ocaml/quicktest/qt.mli index f0edde13a56..15dbb785f28 100644 --- a/ocaml/quicktest/qt.mli +++ b/ocaml/quicktest/qt.mli @@ -50,7 +50,12 @@ module VM : sig end val with_new : - rpc -> API.ref_session -> template:API.ref_VM -> (API.ref_VM -> 'a) -> 'a + rpc + -> API.ref_session + -> template:API.ref_VM + -> ?sr:API.ref_SR + -> (API.ref_VM -> 'a) + -> 'a val dom0_of_host : rpc -> API.ref_session -> API.ref_host -> API.ref_VM (** Return a host's domain zero *) diff --git a/ocaml/quicktest/quicktest_vm_lifecycle.ml b/ocaml/quicktest/quicktest_vm_lifecycle.ml index 88fd9b8d664..b3de6b5b309 100644 --- a/ocaml/quicktest/quicktest_vm_lifecycle.ml +++ b/ocaml/quicktest/quicktest_vm_lifecycle.ml @@ -91,12 +91,18 @@ let one rpc session_id vm test = | Halted -> wait_for_domid (fun domid' -> domid' = -1L) -let test rpc session_id vm_template () = - Qt.VM.with_new rpc session_id ~template:vm_template (fun vm -> +let test rpc session_id sr_info vm_template () = + let sr = sr_info.Qt.sr in + Qt.VM.with_new rpc session_id ~template:vm_template ~sr (fun vm -> List.iter (one rpc session_id vm) all_possible_tests ) let tests () = let open Qt_filter in - [[("VM lifecycle tests", `Slow, test)] |> conn |> vm_template "CoreOS"] + [ + [("VM lifecycle tests", `Slow, test)] + |> conn + |> sr SR.(all |> allowed_operations [`vdi_create]) + |> vm_template "CoreOS" + ] |> List.concat From 316dc9793e2763fb0517d0e96c1a231a696fa30b Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 18 Oct 2023 14:43:10 +0000 Subject: [PATCH 002/149] CP-45888: Add `JsonRpcClient` classes Also update `pom.xml` to remove `org.apache.xmlrpc` packages and add `jackson-databind` and `org.apache.httpcomponents.client5` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/autogen/xen-api/pom.xml | 27 +-- .../com/xensource/xenapi/JsonRpcClient.java | 200 ++++++++++++++++++ .../{Marshalling.java => JsonRpcRequest.java} | 76 +++---- .../com/xensource/xenapi/JsonRpcResponse.java | 59 ++++++ .../xenapi/JsonRpcResponseError.java | 54 +++++ 5 files changed, 352 insertions(+), 64 deletions(-) create mode 100644 ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java rename ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/{Marshalling.java => JsonRpcRequest.java} (51%) create mode 100644 ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponse.java create mode 100644 ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponseError.java diff --git a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml index 471568f1179..c78ec926540 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml +++ b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml @@ -53,29 +53,14 @@ - org.apache.xmlrpc - xmlrpc-client - 3.1.3 + com.fasterxml.jackson.core + jackson-databind + 2.15.1 - org.apache.xmlrpc - xmlrpc-common - 3.1.3 - - - org.apache.ws.commons.util - ws-commons-util - 1.0.2 - - - junit - junit - - - xml-apis - xml-apis - - + org.apache.httpcomponents.client5 + httpclient5 + 5.2.1 diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java new file mode 100644 index 00000000000..8747822672c --- /dev/null +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -0,0 +1,200 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.xensource.xenapi; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.config.ConnectionConfig; +import org.apache.hc.client5.http.config.RequestConfig; +import org.apache.hc.client5.http.cookie.StandardCookieSpec; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.io.entity.StringEntity; + +import java.io.IOException; +import java.net.URL; +import java.net.http.HttpClient; +import java.text.SimpleDateFormat; +import java.util.concurrent.TimeUnit; + +/** + * Provides a JSON-RPC v2.0 client for making remote procedure calls to xapi's backend URL. + *
+ * This class enables the communication to the JSON-RPC backend. The client utilizes the HttpClient class for + * sending HTTP POST requests with JSON payloads and the ObjectMapper class from the Jackson library for + * serialization and deserialization of JSON data. + *
+ * The client can be customised by passing it as a parameter to corresponding constructor, enabling custom + * handling of requests. + *
+ * By default, the timeout for requests is set to 10 minutes (600 seconds). The default timeout for connecting to the + * JSON-RPC backend is set to 5 seconds. + * + * @see HttpClient HttpClient is used to make requests and connect to the backend + * @see ObjectMapper ObjectMapper is used to marshall requests and responses + */ +public class JsonRpcClient { + private static final int DEFAULT_REQUEST_TIMEOUT = 600; + private static final int DEFAULT_CONNECTION_TIMEOUT = 5; + + protected static final int MAX_CONCURRENT_CONNECTIONS = 10; + + protected static final String JSON_BACKEND_PATH = "/jsonrpc"; + + protected final CloseableHttpClient httpClient; + protected final String jsonRpcBackendUrl; + protected final ObjectMapper objectMapper; + protected final int requestTimeout; + + private final RequestConfig defaultRequestConfig = RequestConfig.custom() + .setCookieSpec(StandardCookieSpec.IGNORE) + .build(); + + /** + * Create a JsonRpcClient with default settings. + * + * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. + * @see JsonRpcClient JsonRpcClient for more info on using this class + */ + public JsonRpcClient(URL jsonRpcBackendUrl) { + this(jsonRpcBackendUrl, DEFAULT_REQUEST_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT); + } + + /** + * Create a JsonRpcClient with the option to define the request and connection timeout values. + * + * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. + * @param requestTimeout the timeout value for requests. + * @param connectionTimeout the timeout value for the initial connection to the host. + * @see JsonRpcClient JsonRpcClient for more info on using this class + */ + public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTimeout) { + var connectionConfig = ConnectionConfig + .custom() + .setConnectTimeout(connectionTimeout, TimeUnit.SECONDS) + .build(); + + var connectionManager = new PoolingHttpClientConnectionManager(); + connectionManager.setDefaultConnectionConfig(connectionConfig); + connectionManager.setMaxTotal(MAX_CONCURRENT_CONNECTIONS); + + this.httpClient = HttpClients + .custom() + .setConnectionManager(connectionManager) + .build(); + this.jsonRpcBackendUrl = formatBackendUrl(jsonRpcBackendUrl); + this.requestTimeout = requestTimeout; + this.objectMapper = new ObjectMapper(); + initializeObjectMapperConfiguration(); + } + + /** + * Initialize a JsonRpcClient using a custom HttpClient instance. + * + * @param client the custom HttpClient to use for all requests + * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. + * @param requestTimeout the timeout value for requests. + * @see HttpClient + * @see JsonRpcClient JsonRpcClient for more info on using this class + */ + public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requestTimeout) { + httpClient = client; + + this.requestTimeout = requestTimeout; + this.jsonRpcBackendUrl = formatBackendUrl(jsonRpcBackendUrl); + + this.objectMapper = new ObjectMapper(); + initializeObjectMapperConfiguration(); + } + + /** + * Send a method call to xapi's backend. You need to provide the type of the data returned by a successful response. + * + * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password + * @param methodParameters the parameters of the method call + * @param responseTypeReference the type of the response, wrapped with a TypeReference + * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records + * @return a JsonRpcResponse object. If its error field is empty, the response was successful. + * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON + * @throws IOException if an I/O error occurs when sending or receiving + */ + public JsonRpcResponse sendRequest(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws IOException { + var requestBody = objectMapper + .writeValueAsString(new JsonRpcRequest(methodCall, methodParameters)); + + var requestEntity = new StringEntity(requestBody, ContentType.APPLICATION_JSON); + + var requestConfig = RequestConfig.copy(defaultRequestConfig) + .setConnectionRequestTimeout(this.requestTimeout, TimeUnit.SECONDS) + .build(); + + var request = new HttpPost(this.jsonRpcBackendUrl); + request.setConfig(requestConfig); + request.setEntity(requestEntity); + + return httpClient.execute(request, response -> { + try (response) { + var typeFactory = objectMapper.getTypeFactory(); + var responseObjectType = typeFactory.constructType(responseTypeReference.getType()); + var type = typeFactory.constructParametricType(JsonRpcResponse.class, responseObjectType); + + var responseContent = response.getEntity().getContent(); + return objectMapper.readValue(responseContent, type); + } + }); + } + + /** + * Helper method to initialize jackson's ObjectMapper. + */ + private void initializeObjectMapperConfiguration() { + this.objectMapper.setDateFormat(new SimpleDateFormat("yyyyMMdd'T'HH:mm:ss'Z'")); + } + + /** + * Format input URL to the form protocol://host/jsonrpc + * + * @param url the input URL to format + * @return a string version of a valid xen-api backend URL + */ + private String formatBackendUrl(URL url) { + // We only replace it when it's empty. + // If the user purposely set the path + // we use the given value even if incorrect + if (!url.getPath().isEmpty()) { + return url.getProtocol() + "://" + url.getHost() + JSON_BACKEND_PATH; + } + return url.toString(); + } +} diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Marshalling.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcRequest.java similarity index 51% rename from ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Marshalling.java rename to ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcRequest.java index cc7177a92ea..fa49528c205 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Marshalling.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcRequest.java @@ -1,18 +1,18 @@ /* * Copyright (c) Cloud Software Group, Inc. - * + * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: - * + * * 1) Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. - * + * * 2) Redistributions in binary form must reproduce the above * copyright notice, this list of conditions and the following * disclaimer in the documentation and/or other materials * provided with the distribution. - * + * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS @@ -29,48 +29,38 @@ package com.xensource.xenapi; -import java.util.*; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; /** - * Marshalls Java types onto the wire. - * Does not cope with records. Use individual record.toMap() + * Represents the payload of a request sent to a + * JSON-RPC v2.0 backend. */ -public final class Marshalling { - /** - * Converts Integers to Strings - * and Sets to Lists recursively. - */ - public static Object toXMLRPC(Object o) { - if (o instanceof String || - o instanceof Boolean || - o instanceof Double || - o instanceof Date) { - return o; - } else if (o instanceof Long) { - return o.toString(); - } else if (o instanceof Map) { - Map result = new HashMap(); - Map m = (Map)o; - for (Object k : m.keySet()) - { - result.put(toXMLRPC(k), toXMLRPC(m.get(k))); - } - return result; - } else if (o instanceof Set) { - List result = new ArrayList(); - for (Object e : ((Set)o)) - { - result.add(toXMLRPC(e)); - } - return result; - } else if (o instanceof XenAPIObject) { - return ((XenAPIObject) o).toWireString(); - } else if (o instanceof Enum) { - return o.toString(); - }else if (o == null){ - return ""; - } else { - throw new RuntimeException ("=============don't know how to marshall:({[" + o + "]})"); +public class JsonRpcRequest { + @JsonProperty("jsonrpc") + public String jsonRpc; + public int id; + public String method; + @JsonProperty("params") + public Object[] parameters; + + public JsonRpcRequest(String method, Object[] parameters) { + this.method = method; + this.parameters = parameters; + this.jsonRpc = "2.0"; + this.id = 1; + } + + @Override + public String toString() { + try { + var mapper = new ObjectMapper(); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + return mapper.writeValueAsString(this); + } catch (JsonProcessingException ex) { + return "Error while processing object. Could not serialize as JSON."; } } } diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponse.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponse.java new file mode 100644 index 00000000000..11e2c152474 --- /dev/null +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponse.java @@ -0,0 +1,59 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.xensource.xenapi; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +/** + * Represents the payload of responses returned from a + * JSON-RPC v2.0 backend. + * @param The type of the response's result + */ +public class JsonRpcResponse { + @JsonProperty("jsonrpc") + public String jsonRpc; + public int id; + public T result; + public JsonRpcResponseError error; + + @Override + public String toString() { + try { + var mapper = new ObjectMapper(); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + return mapper.writeValueAsString(this); + } catch (JsonProcessingException ex) { + return "Error while processing object. Could not serialize as JSON"; + } + } +} diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponseError.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponseError.java new file mode 100644 index 00000000000..80cbabcb582 --- /dev/null +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcResponseError.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.xensource.xenapi; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +/** + * Represents the structure of an error returned by a + * JSON-RPC v2.0 backend. Does not apply to JSON-RPC v1.0. + */ +public class JsonRpcResponseError { + public int code; + public String message; + public String[] data; + + public String toString() { + try { + var mapper = new ObjectMapper(); + mapper.enable(SerializationFeature.INDENT_OUTPUT); + return mapper.writeValueAsString(this); + } catch (JsonProcessingException ex) { + return "Error while processing object. Could not serialize as JSON."; + } + } +} From 2ec9542c0fd97a763daf5358eeeee625bbda8b9a Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 18 Oct 2023 14:43:10 +0000 Subject: [PATCH 003/149] CP-45888: Update `Connection.java` to use new `JsonRpcClient` Signed-off-by: Danilo Del Busso --- .../java/com/xensource/xenapi/Connection.java | 274 +++++++++--------- 1 file changed, 130 insertions(+), 144 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index ef5b137b315..f568c452894 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -29,228 +29,214 @@ package com.xensource.xenapi; -import java.net.URL; -import java.util.Map; -import java.util.TimeZone; - -import org.apache.xmlrpc.XmlRpcException; -import org.apache.xmlrpc.client.XmlRpcClient; -import org.apache.xmlrpc.client.XmlRpcClientConfigImpl; -import org.apache.xmlrpc.client.XmlRpcHttpClientConfig; - +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; import com.xensource.xenapi.Types.BadServerResponse; import com.xensource.xenapi.Types.XenAPIException; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; + +import java.io.IOException; +import java.net.URL; /** - * Represents a connection to a XenServer. Creating a new instance of this class initialises a new XmlRpcClient that is - * then used by all method calls: each method call in xenapi takes a Connection as a parameter, composes an XMLRPC + * Represents a connection to a XenServer. Creating a new instance of this class initialises a new JsonRpcClient that is + * then used by all method calls: each method call in xen-api takes a Connection as a parameter, composes a JSON-RPC * method call, and dispatches it on the Connection's client via the dispatch method. */ -public class Connection -{ +public class Connection { + private final JsonRpcClient client; private APIVersion apiVersion; - - /** - * Default reply timeout for xml-rpc calls in seconds - */ - protected static final int DEFAULT_REPLY_TIMEOUT = 600; - /** - * Default connection timeout for xml-rpc calls in seconds + * The opaque reference to the session used by this connection */ - protected static final int DEFAULT_CONNECTION_TIMEOUT = 5; + private String sessionReference; /** - * Updated when Session.login_with_password() is called. + * Creates a connection to a particular server using a custom implementation of the JsonRpcClient. + *

+ * Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, + * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. + *

+ * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually + * logging out the Session. + * + * @param jsonRpcClient The JsonRpcClient used to connect to the JSON-RPC backed. */ - public APIVersion getAPIVersion() - { - return apiVersion; + public Connection(JsonRpcClient jsonRpcClient) { + this.client = jsonRpcClient; } /** - * The opaque reference to the session used by this connection - */ - private String sessionReference; - - /** - * As seen by the xmlrpc library. From our point of view it's a server. + * Creates a connection to a particular server using a given url. This object can then be passed + * in to any other API calls. + *

+ * Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, + * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. + *

+ * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually + * logging out the Session. + * + * @param httpClient The HttpClient used to make calls, this will be used by JsonRpcClient for handling requests + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param requestTimeout The reply timeout for JSON-RPC calls in seconds */ - private final XmlRpcClient client; + public Connection(CloseableHttpClient httpClient, URL url, int requestTimeout) { + this.client = new JsonRpcClient(httpClient, url, requestTimeout); + } /** * Creates a connection to a particular server using a given url. This object can then be passed * in to any other API calls. - * + *

* Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. - * + *

* When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. - * - * This constructor uses the default values of the reply and connection timeouts for the xmlrpc calls + *

+ * This constructor uses the default values of the reply and connection timeouts for the JSON-RPC calls * (600 seconds and 5 seconds respectively). * - * @param url The URL of the server to connect to + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. */ - public Connection(URL url) - { - this.client = getClientFromURL(url, DEFAULT_REPLY_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT); + public Connection(URL url) { + this.client = new JsonRpcClient(url); } /** * Creates a connection to a particular server using a given url. This object can then be passed * in to any other API calls. - * + *

* Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. - * + *

* When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. * - * @param url The URL of the server to connect to - * @param replyTimeout The reply timeout for xml-rpc calls in seconds - * @param connTimeout The connection timeout for xml-rpc calls in seconds + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param requestTimeout The reply timeout for JSON-RPC calls in seconds + * @param connectionTimeout The connection timeout for JSON-RPC calls in seconds */ - public Connection(URL url, int replyTimeout, int connTimeout) - { - this.client = getClientFromURL(url, replyTimeout, connTimeout); + public Connection(URL url, int requestTimeout, int connectionTimeout) { + this.client = new JsonRpcClient(url, requestTimeout, connectionTimeout); } - /** * Creates a connection to a particular server using a given url. This object can then be passed * in to any other API calls. - * - * This constructor uses the default values of the reply and connection timeouts for the xmlrpc calls + *

+ * Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, + * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. + *

+ * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually + * logging out the Session. + *

+ * This constructor uses the default values of the reply and connection timeouts for the JSON-RPC calls * (600 seconds and 5 seconds respectively). * - * @param url The URL of the server to connect to + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. * @param sessionReference A reference to a logged-in Session. Any method calls on this - * Connection will use it. This constructor does not call Session.loginWithPassword, and dispose() on the resulting - * Connection object does not call Session.logout. The programmer is responsible for ensuring the Session is logged - * in and out correctly. + * Connection will use it. This constructor does not call Session.loginWithPassword, and dispose() on the resulting + * Connection object does not call Session.logout. The programmer is responsible for ensuring the Session is logged + * in and out correctly. */ - public Connection(URL url, String sessionReference) - { - this.client = getClientFromURL(url, DEFAULT_REPLY_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT); + public Connection(URL url, String sessionReference) { + this.client = new JsonRpcClient(url); this.sessionReference = sessionReference; } /** * Creates a connection to a particular server using a given url. This object can then be passed * in to any other API calls. + *

+ * Note this constructor does NOT call Session.loginWithPassword; the programmer is responsible for calling it, + * passing the Connection as a parameter. No attempt to connect to the server is made until login is called. + *

+ * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually + * logging out the Session. * - * @param url The URL of the server to connect to - * @param sessionReference A reference to a logged-in Session. Any method calls on this Connection will use it. - * This constructor does not call Session.loginWithPassword, and dispose() on the resulting - * Connection object does not call Session.logout. The programmer is responsible for - * ensuring the Session is logged in and out correctly. - * @param replyTimeout The reply timeout for xml-rpc calls in seconds - * @param connTimeout The connection timeout for xml-rpc calls in seconds + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param sessionReference A reference to a logged-in Session. Any method calls on this Connection will use it. + * This constructor does not call Session.loginWithPassword, and dispose() on the resulting + * Connection object does not call Session.logout. The programmer is responsible for + * ensuring the Session is logged in and out correctly. + * @param requestTimeout The reply timeout for JSON-RPC calls in seconds + * @param connectionTimeout The connection timeout for JSON-RPC calls in seconds */ - public Connection(URL url, String sessionReference, int replyTimeout, int connTimeout) - { - this.client = getClientFromURL(url, replyTimeout, connTimeout); + public Connection(URL url, String sessionReference, int requestTimeout, int connectionTimeout) { + this.client = new JsonRpcClient(url, requestTimeout, connectionTimeout); this.sessionReference = sessionReference; } - private XmlRpcClientConfigImpl config = new XmlRpcClientConfigImpl(); - - public XmlRpcClientConfigImpl getConfig() - { - return config; + /** + * Updated when Session.login_with_password() is called. + */ + public APIVersion getAPIVersion() { + return apiVersion; } - private XmlRpcClient getClientFromURL(URL url, int replyWait, int connWait) - { - config.setTimeZone(TimeZone.getTimeZone("UTC")); - config.setServerURL(url); - config.setReplyTimeout(replyWait * 1000); - config.setConnectionTimeout(connWait * 1000); - XmlRpcClient client = new XmlRpcClient(); - client.setConfig(config); - return client; + private void setAPIVersion(Session session) throws IOException { + try { + long major = session.getThisHost(this).getAPIVersionMajor(this); + long minor = session.getThisHost(this).getAPIVersionMajor(this); + apiVersion = APIVersion.fromMajorMinor(major, minor); + } catch (BadServerResponse exn) { + apiVersion = APIVersion.UNKNOWN; + } } /* * Because the binding calls are constructing their own parameter lists, they need to be able to get to * the session reference directly. This is all rather ugly and needs redone - * Changed to public to allow easier integration with HTTP-level streaming interface, - * see CA-15447 + * CA-15447: Changed to public in order to allow easier integration with HTTP-level streaming interface, */ - public String getSessionReference() - { + public String getSessionReference() { return this.sessionReference; } /** - * The (auto-generated parts of) the bindings dispatch XMLRPC calls on this Connection's client through this method. + * Send a method call to xapi's backend. You need to provide the type of the data returned by a successful response. + * + * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password + * @param methodParameters the methodParameters of the method call + * @param responseTypeReference the type of the response, wrapped with a TypeReference + * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records + * @return The result of the call with the type specified under T. + * @throws XenAPIException if the call failed. + * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON + * @throws IOException if an I/O error occurs when sending or receiving */ - protected Map dispatch(String method_call, Object[] method_params) throws XmlRpcException, XenAPIException - { - Map response = (Map) client.execute(method_call, method_params); + public T dispatch(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws XenAPIException, JsonProcessingException, IOException { + var result = client.sendRequest(methodCall, methodParameters, responseTypeReference); + if (result.error != null) { + throw new XenAPIException(String.valueOf(result.error)); + } - if (method_call.equals("session.login_with_password") && - response.get("Status").equals("Success")) - { - Session session = Types.toSession(response.get("Value")); + if (methodCall.equals("session.login_with_password")) { + var session = ((Session) result.result); sessionReference = session.ref; setAPIVersion(session); - } - else if (method_call.equals("session.slave_local_login_with_password") && - response.get("Status").equals("Success")) - { - sessionReference = Types.toSession(response.get("Value")).ref; + } else if (methodCall.equals("session.slave_local_login_with_password")) { + var session = ((Session) result.result); + sessionReference = session.ref; apiVersion = APIVersion.latest(); } - else if (method_call.equals("session.logout")) - { - // Work around a bug in XenServer 5.0 and below. - // session.login_with_password should have rejected us with - // HOST_IS_SLAVE, but instead we don't find out until later. - // We don't want to leak the session, so we need to log out - // this session from the master instead. - if (response.get("Status").equals("Failure")) - { - Object[] error = (Object[]) response.get("ErrorDescription"); - if (error.length == 2 && error[0].equals("HOST_IS_SLAVE")) - { - try - { - XmlRpcHttpClientConfig clientConfig = (XmlRpcHttpClientConfig)client.getClientConfig(); - URL client_url = clientConfig.getServerURL(); - URL masterUrl = new URL(client_url.getProtocol(), (String)error[1], client_url.getPort(), client_url.getFile()); - - Connection tmp_conn = new Connection(masterUrl, sessionReference, clientConfig.getReplyTimeout(), clientConfig.getConnectionTimeout()); - - Session.logout(tmp_conn); - } - catch (Exception ex) - { - // Ignore - } - } - } - - this.sessionReference = null; - } - return Types.checkResponse(response); + return result.result; } - - private void setAPIVersion(Session session) throws XenAPIException, XmlRpcException - { - try - { - long major = session.getThisHost(this).getAPIVersionMajor(this); - long minor = session.getThisHost(this).getAPIVersionMinor(this); - apiVersion = APIVersion.fromMajorMinor(major, minor); - } - catch (BadServerResponse exn) - { - apiVersion = APIVersion.UNKNOWN; - } + /** + * Send a method call to xapi's backend. To be used with methods without a return type + * + * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password + * @param methodParameters the methodParameters of the method call + * @throws XenAPIException if the call failed. + * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON + * @throws IOException if an I/O error occurs when sending or receiving + */ + public void dispatch(String methodCall, Object[] methodParameters) throws XenAPIException, JsonProcessingException, IOException { + var typeReference = new TypeReference() { + }; + this.dispatch(methodCall, methodParameters, typeReference); } } From 31062352aebd0a7e6480bfdde9aae7947a2749bf Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 18 Oct 2023 14:43:10 +0000 Subject: [PATCH 004/149] CP-45888: Update Java SDK generation to use `JsonRpcClient` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 195 +++++++++++++------------------------ 1 file changed, 70 insertions(+), 125 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 8efaafed4f3..b30873670bf 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -142,7 +142,7 @@ let rec get_java_type ty = Hashtbl.replace enums name ls ; sprintf "Types.%s" (class_case name) | Set t1 -> - sprintf "Set<%s>" (get_java_type t1) + sprintf "HashSet<%s>" (get_java_type t1) | Map (t1, t2) -> sprintf "Map<%s, %s>" (get_java_type t1) (get_java_type t2) | Ref x -> @@ -278,10 +278,13 @@ let gen_method file cls message params async_version = ( "BadServerResponse" , "Thrown if the response from the server contains an invalid status." ) - ; ("XenAPIException", "Thrown if the call failed.") - ; ( "XmlRpcException" - , "Thrown if the result of an asynchronous call could not be parsed." + ; ("XenAPIException", "if the call failed.") + ; ( "JsonProcessingException" + , "if the request's payload or the response's payload cannot be written \ + or read as valid JSON." ) + ; ("IOException", "if an I/O error occurs when sending or receiving.") + ; ("InterruptedException", " if the operation is interrupted.") ] in let publishInfo = get_published_info_message message cls in @@ -346,14 +349,14 @@ let gen_method file cls message params async_version = fprintf file " %s {\n" (String.concat ",\n " all_errors) ; if async_version then - fprintf file " String method_call = \"Async.%s.%s\";\n" + fprintf file " String methodCall = \"Async.%s.%s\";\n" message.msg_obj_name message.msg_name else - fprintf file " String method_call = \"%s.%s\";\n" - message.msg_obj_name message.msg_name ; + fprintf file " String methodCall = \"%s.%s\";\n" message.msg_obj_name + message.msg_name ; if message.msg_session then - fprintf file " String session = c.getSessionReference();\n" + fprintf file " String sessionReference = c.getSessionReference();\n" else () ; @@ -371,38 +374,32 @@ let gen_method file cls message params async_version = ) record_params ; - fprintf file " Object[] method_params = {" ; + fprintf file " Object[] methodParameters = {" ; let methodParamsList = if message.msg_session then - "session" :: get_method_params_for_xml message params + "sessionReference" :: get_method_params_for_xml message params else get_method_params_for_xml message params in output_string file - (String.concat ", " - (List.map - (fun s -> sprintf "Marshalling.toXMLRPC(%s)" s) - methodParamsList - ) - ) ; + (String.concat ", " (List.map (fun s -> sprintf "%s" s) methodParamsList)) ; fprintf file "};\n" ; - fprintf file - " Map response = c.dispatch(method_call, method_params);\n" ; - ( if async_version then ( - fprintf file " Object result = response.get(\"Value\");\n" ; - fprintf file " return Types.toTask(result);\n" - ) else - match message.msg_result with - | None -> - fprintf file "" - | Some _ -> - fprintf file " Object result = response.get(\"Value\");\n" ; - gen_method_return file cls message - ) ; + if message.msg_result != None || async_version then + fprintf file " var typeReference = new TypeReference<%s>(){};\n" + (if async_version then "Task" else return_type) ; + + let last_statement = + match message.msg_result with + | None when not async_version -> + " c.dispatch(methodCall, methodParameters);\n" + | _ -> + " return c.dispatch(methodCall, methodParameters, typeReference);\n" + in + fprintf file "%s" last_statement ; fprintf file " }\n\n" @@ -430,14 +427,14 @@ let gen_method_and_asynchronous_counterpart file cls message = let gen_record_field file prefix field cls = let ty = get_java_type field.ty in - let name = - camel_case (String.concat "_" (List.rev (field.field_name :: prefix))) - in + let full_name = String.concat "_" (List.rev (field.field_name :: prefix)) in + let name = camel_case full_name in let publishInfo = get_published_info_field field cls in fprintf file " /**\n" ; fprintf file " * %s\n" (escape_xml field.field_description) ; if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; fprintf file " */\n" ; + fprintf file " @JsonProperty(\"%s\")\n" full_name ; fprintf file " public %s %s;\n" ty name let rec gen_record_namespace file prefix (name, contents) cls = @@ -571,16 +568,15 @@ let gen_class cls folder = print_license file ; fprintf file "package com.xensource.xenapi;\n\n\ + import com.fasterxml.jackson.annotation.JsonProperty;\n\ + import com.fasterxml.jackson.core.JsonProcessingException;\n\ + import com.fasterxml.jackson.core.type.TypeReference;\n\ import com.xensource.xenapi.Types.BadServerResponse;\n\ import com.xensource.xenapi.Types.XenAPIException;\n\n\ import java.io.PrintWriter;\n\ import java.io.StringWriter;\n\ - import java.util.Date;\n\ - import java.util.HashMap;\n\ - import java.util.LinkedHashSet;\n\ - import java.util.Map;\n\ - import java.util.Set;\n\n\ - import org.apache.xmlrpc.XmlRpcException;\n\n" ; + import java.util.*;\n\ + import java.io.IOException;\n\n" ; fprintf file "/**\n" ; fprintf file " * %s\n" cls.description ; if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; @@ -753,7 +749,8 @@ let rec gen_marshall_body file = function let ty_name = get_java_type ty in let marshall_fn = get_marshall_function ty in fprintf file " Object[] items = (Object[]) object;\n" ; - fprintf file " Set<%s> result = new LinkedHashSet<>();\n" ty_name ; + fprintf file " HashSet<%s> result = new LinkedHashSet<>();\n" + ty_name ; fprintf file " for(Object item: items) {\n" ; fprintf file " %s typed = %s(item);\n" ty_name marshall_fn ; fprintf file " result.add(typed);\n" ; @@ -767,7 +764,7 @@ let rec gen_marshall_body file = function fprintf file " Map map = (Map)object;\n" ; fprintf file " Map<%s,%s> result = new HashMap<>();\n" ty_name ty_name' ; - fprintf file " Set entries = map.entrySet();\n" ; + fprintf file " HashSet entries = map.entrySet();\n" ; fprintf file " for(Map.Entry entry: entries) {\n" ; fprintf file " %s key = %s(entry.getKey());\n" ty_name marshall_fn ; @@ -821,13 +818,20 @@ let gen_enum file name ls = let final_description = global_replace (regexp_string "\n") "\n * " escaped_description in - " /**\n" - ^ " * " - ^ final_description - ^ "\n" - ^ " */\n" - ^ " " - ^ enum_of_wire name + let comment = + " /**\n" + ^ " * " + ^ final_description + ^ "\n" + ^ " */\n" + in + let json_property = + if name != "UNRECOGNIZED" then + "@JsonProperty(\"" ^ name ^ "\")" + else + "@JsonEnumDefaultValue" + in + comment ^ " " ^ json_property ^ "\n" ^ " " ^ enum_of_wire name in fprintf file "%s" (String.concat ",\n" (List.map to_member_declaration ls)) ; fprintf file ";\n" ; @@ -888,15 +892,15 @@ let gen_method_error_throw file name error = ) in - fprintf file " if (ErrorDescription[0].equals(\"%s\"))\n" name ; + fprintf file " if (errorName.equals(\"%s\"))\n" name ; fprintf file " {\n" ; (* Prepare the parameters to the Exception constructor *) List.iter (fun i -> fprintf file - " String p%i = ErrorDescription.length > %i ? \ - ErrorDescription[%i] : \"\";\n" + " String p%i = errorData.length > %i ? \ + errorData[%i] : \"\";\n" i i i ) (range (List.length error.err_params)) ; @@ -910,19 +914,12 @@ let gen_types_class folder = print_license file ; fprintf file "package com.xensource.xenapi;\n\n\ - import java.util.Date;\n\ - import java.util.Map;\n\ - import java.util.HashMap;\n\ - import java.util.Set;\n\ - import java.util.LinkedHashSet;\n\ - import java.io.IOException;\n\n\ - import java.util.regex.Pattern;\n\ - import java.util.regex.Matcher;\n\n\ - import org.apache.xmlrpc.XmlRpcException;\n\n\ + import java.util.Map;\n\n\ + import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;\n\ + import com.fasterxml.jackson.annotation.JsonProperty;\n\ + import java.io.IOException;\n\n\ /**\n\ - \ * This class holds vital marshalling functions, enum types and exceptions.\n\ - \ *\n\ - \ * @author Cloud Software Group, Inc.\n\ + \ * This class holds enum types and exceptions.\n\ \ */\n\ public class Types\n\ {\n\ @@ -937,18 +934,6 @@ let gen_types_class folder = \ Map toMap();\n\ \ }\n\n\ \ /**\n\ - \ * Helper method.\n\ - \ */\n\ - \ private static String[] ObjectArrayToStringArray(Object[] objArray)\n\ - \ {\n\ - \ String[] result = new String[objArray.length];\n\ - \ for (int i = 0; i < objArray.length; i++)\n\ - \ {\n\ - \ result[i] = (String) objArray[i];\n\ - \ }\n\ - \ return result;\n\ - \ }\n\n\ - \ /**\n\ \ * Base class for all XenAPI Exceptions\n\ \ */\n\ \ public static class XenAPIException extends IOException {\n\ @@ -993,30 +978,11 @@ let gen_types_class folder = \ */\n\ \ public static class BadServerResponse extends XenAPIException\n\ \ {\n\ - \ public BadServerResponse(Map response)\n\ - \ {\n\ - \ super(ObjectArrayToStringArray((Object[]) \ - response.get(\"ErrorDescription\")));\n\ - \ }\n\ - \ }\n\n\ - \ public static class BadAsyncResult extends XenAPIException\n\ - \ {\n\ - \ public final String result;\n\n\ - \ public BadAsyncResult(String result)\n\ + \ public BadServerResponse(JsonRpcResponseError responseError)\n\ \ {\n\ - \ super(result);\n\ - \ this.result = result;\n\ + \ super(String.valueOf(responseError));\n\ \ }\n\ \ }\n\n\ - \ private static String parseResult(String result) throws BadAsyncResult\n\ - \ {\n\ - \ Pattern pattern = Pattern.compile(\"(.*)\");\n\ - \ Matcher matcher = pattern.matcher(result);\n\ - \ if (!matcher.find() || matcher.groupCount() != 1) {\n\ - \ throw new Types.BadAsyncResult(\"Can't interpret: \" + result);\n\ - \ }\n\n\ - \ return matcher.group(1);\n\ - \ }\n\ \ " ; fprintf file @@ -1026,48 +992,27 @@ let gen_types_class folder = \ * returned an invalid response, throws a BadServerResponse. \ Otherwise, returns the server response as passed in.\n\ \ */\n\ - \ static Map checkResponse(Map response) throws XenAPIException, \ + \ public static void checkError(JsonRpcResponseError response) throws XenAPIException, \ BadServerResponse\n\ \ {\n\ - \ if (response.get(\"Status\").equals(\"Success\"))\n\ - \ {\n\ - \ return response;\n\ - \ }\n\n\ - \ if (response.get(\"Status\").equals(\"Failure\"))\n\ - \ {\n\ - \ String[] ErrorDescription = \ - ObjectArrayToStringArray((Object[]) response.get(\"ErrorDescription\"));\n\n" ; + \ var errorData = response.data; + \ if(errorData.length == 0){ + \ throw new BadServerResponse(response); + \ } + \ var errorName = errorData[0];\n\n" ; Hashtbl.iter (gen_method_error_throw file) Datamodel.errors ; fprintf file "\n\ - \ // An unknown error occurred\n\ - \ throw new Types.XenAPIException(ErrorDescription);\n\ - \ }\n\n\ - \ throw new BadServerResponse(response);\n\ - \ }\n\n" ; + \ // An unknown error occurred\n\ + \ throw new Types.XenAPIException(errorData);\n\ + \ }\n\n" ; gen_enums file ; fprintf file "\n" ; Hashtbl.iter (gen_error file) Datamodel.errors ; fprintf file "\n" ; - TypeSet.iter (gen_marshall_func file) !types ; - fprintf file "\n" ; - TypeSet.iter (gen_task_result_func file) !types ; - fprintf file - "\n\ - \ public static EventBatch toEventBatch(Object object) {\n\ - \ if (object == null) {\n\ - \ return null;\n\ - \ }\n\n\ - \ Map map = (Map) object;\n\ - \ EventBatch batch = new EventBatch();\n\ - \ batch.token = toString(map.get(\"token\"));\n\ - \ batch.validRefCounts = map.get(\"valid_ref_counts\");\n\ - \ batch.events = toSetOfEventRecord(map.get(\"events\"));\n\ - \ return batch;\n\ - \ }" ; fprintf file "}\n" (* Now run it *) From 93b9839c755d6b8db454c3550067931062c2106d Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 1 Nov 2023 15:23:38 +0000 Subject: [PATCH 005/149] CP-45888: Remove unsed functions from Java's `main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 208 ------------------------------------- 1 file changed, 208 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index b30873670bf..6a77a157b97 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -159,36 +159,6 @@ let switch_enum = let _ = get_java_type switch_enum -(*Helper function for get_marshall_function*) -let rec get_marshall_function_rec = function - | SecretString | String -> - "String" - | Int -> - "Long" - | Float -> - "Double" - | Bool -> - "Boolean" - | DateTime -> - "Date" - | Enum (name, _) -> - class_case name - | Set t1 -> - sprintf "SetOf%s" (get_marshall_function_rec t1) - | Map (t1, t2) -> - sprintf "MapOf%s%s" - (get_marshall_function_rec t1) - (get_marshall_function_rec t2) - | Ref ty -> - class_case ty (* We want to hide all refs *) - | Record ty -> - sprintf "%sRecord" (class_case ty) - | Option ty -> - get_marshall_function_rec ty - -(*get_marshall_function (Set(Map(Float,Bool)));; -> "toSetOfMapOfDoubleBoolean"*) -let get_marshall_function ty = "to" ^ get_marshall_function_rec ty - (* Generate the methods *) let get_java_type_or_void = function @@ -240,22 +210,6 @@ let get_method_params_for_xml message params = else "this.ref" :: List.map f params -let gen_method_return_cast message = - match message.msg_result with - | None -> - sprintf "" - | Some (ty, _) -> - sprintf " Types.%s(result)" (get_marshall_function ty) - -let gen_method_return file cls message = - if - String.lowercase_ascii cls.name = "event" - && String.lowercase_ascii message.msg_name = "from" - then - fprintf file " return Types.toEventBatch(result);\n" - else - fprintf file " return%s;\n" (gen_method_return_cast message) - let rec range = function 0 -> [] | i -> range (i - 1) @ [i] (* Here is the main method generating function.*) @@ -284,7 +238,6 @@ let gen_method file cls message params async_version = or read as valid JSON." ) ; ("IOException", "if an I/O error occurs when sending or receiving.") - ; ("InterruptedException", " if the operation is interrupted.") ] in let publishInfo = get_published_info_message message cls in @@ -644,167 +597,6 @@ let gen_class cls folder = fprintf file "}" ; close_out file -(* Generate Marshalling Class *) - -(*This generates the special case code for marshalling the snapshot field in an Event.Record*) -let generate_snapshot_hack file = - fprintf file "\n" ; - fprintf file "\n" ; - fprintf file " Object a,b;\n" ; - fprintf file " a=map.get(\"snapshot\");\n" ; - fprintf file " switch(%s(record.clazz))\n" - (get_marshall_function switch_enum) ; - fprintf file " {\n" ; - List.iter - (fun x -> - fprintf file " case %17s: b = %25s(a); break;\n" - (String.uppercase_ascii x) - (get_marshall_function (Record x)) - ) - (List.map - (fun x -> x.name) - (List.filter (fun x -> not (class_is_empty x)) classes) - ) ; - fprintf file - " default: throw new RuntimeException(\"Internal error in \ - auto-generated code whilst unmarshalling event snapshot\");\n" ; - fprintf file " }\n" ; - fprintf file " record.snapshot = b;\n" - -let gen_marshall_record_field file prefix field = - let ty = get_marshall_function field.ty in - let name = String.concat "_" (List.rev (field.field_name :: prefix)) in - let name' = camel_case name in - fprintf file " record.%s = %s(map.get(\"%s\"));\n" name' ty name - -let rec gen_marshall_record_namespace file prefix (name, contents) = - List.iter (gen_marshall_record_contents file (name :: prefix)) contents - -and gen_marshall_record_contents file prefix = function - | Field f -> - gen_marshall_record_field file prefix f - | Namespace (n, cs) -> - gen_marshall_record_namespace file prefix (n, cs) ; - () - -(*Every type which may be returned by a function may also be the result of the*) -(* corresponding asynchronous task. We therefore need to generate corresponding*) -(* marshalling functions which can take the raw xml of the tasks result field*) -(* and turn it into the corresponding type. Luckily, the only things returned by*) -(* asynchronous tasks are object references and strings, so rather than implementing*) -(* the general recursive structure we'll just make one for each of the classes*) -(* that's been registered as a marshall-needing type*) - -let generate_reference_task_result_func file clstr = - fprintf file - " public static %s to%s(Task task, Connection connection) throws \ - XenAPIException, BadServerResponse, XmlRpcException, BadAsyncResult{\n" - clstr clstr ; - fprintf file - " return Types.to%s(parseResult(task.getResult(connection)));\n" - clstr ; - fprintf file " }\n" ; - fprintf file "\n" - -let gen_task_result_func file = function - | Ref ty -> - generate_reference_task_result_func file (class_case ty) - | _ -> - () - -(*don't generate for complicated types. They're not needed.*) - -let rec gen_marshall_body file = function - | SecretString | String -> - fprintf file " return (String) object;\n" - | Int -> - fprintf file " return Long.valueOf((String) object);\n" - | Float -> - fprintf file " return (Double) object;\n" - | Bool -> - fprintf file " return (Boolean) object;\n" - | DateTime -> - fprintf file - " try {\n\ - \ return (Date) object;\n\ - \ } catch (ClassCastException e){\n\ - \ //Occasionally the date comes back as an ocaml float \ - rather than\n\ - \ //in the xmlrpc format! Catch this and convert.\n\ - \ return (new Date((long) (1000*Double.parseDouble((String) \ - object))));\n\ - \ }\n" - | Ref ty -> - fprintf file " return new %s((String) object);\n" (class_case ty) - | Enum (name, _) -> - fprintf file " try {\n" ; - fprintf file - " return %s.valueOf(((String) \ - object).toUpperCase().replace('-','_'));\n" - (class_case name) ; - fprintf file " } catch (IllegalArgumentException ex) {\n" ; - fprintf file " return %s.UNRECOGNIZED;\n" (class_case name) ; - fprintf file " }\n" - | Set ty -> - let ty_name = get_java_type ty in - let marshall_fn = get_marshall_function ty in - fprintf file " Object[] items = (Object[]) object;\n" ; - fprintf file " HashSet<%s> result = new LinkedHashSet<>();\n" - ty_name ; - fprintf file " for(Object item: items) {\n" ; - fprintf file " %s typed = %s(item);\n" ty_name marshall_fn ; - fprintf file " result.add(typed);\n" ; - fprintf file " }\n" ; - fprintf file " return result;\n" - | Map (ty, ty') -> - let ty_name = get_java_type ty in - let ty_name' = get_java_type ty' in - let marshall_fn = get_marshall_function ty in - let marshall_fn' = get_marshall_function ty' in - fprintf file " Map map = (Map)object;\n" ; - fprintf file " Map<%s,%s> result = new HashMap<>();\n" ty_name - ty_name' ; - fprintf file " HashSet entries = map.entrySet();\n" ; - fprintf file " for(Map.Entry entry: entries) {\n" ; - fprintf file " %s key = %s(entry.getKey());\n" ty_name - marshall_fn ; - fprintf file " %s value = %s(entry.getValue());\n" ty_name' - marshall_fn' ; - fprintf file " result.put(key, value);\n" ; - fprintf file " }\n" ; - fprintf file " return result;\n" - | Record ty -> - let contents = Hashtbl.find records ty in - let cls_name = class_case ty in - fprintf file - " Map map = (Map) object;\n" ; - fprintf file " %s.Record record = new %s.Record();\n" cls_name - cls_name ; - List.iter (gen_marshall_record_contents file []) contents ; - (*Event.Record needs a special case to handle snapshots*) - if ty = "event" then generate_snapshot_hack file ; - fprintf file " return record;\n" - | Option ty -> - gen_marshall_body file ty - -let rec gen_marshall_func file ty = - match ty with - | Option x -> - if TypeSet.mem x !types then - () - else - gen_marshall_func file ty - | _ -> - let type_string = get_java_type ty in - let fn_name = get_marshall_function ty in - fprintf file " public static %s %s(Object object) {\n" type_string - fn_name ; - fprintf file " if (object == null) {\n" ; - fprintf file " return null;\n" ; - fprintf file " }\n" ; - gen_marshall_body file ty ; - fprintf file " }\n\n" - let gen_enum file name ls = let name = class_case name in let ls = From 9bed681e684cde4af9d4be3ab7c779e312b46639 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 6 Nov 2023 11:29:56 +0000 Subject: [PATCH 006/149] CP-45888: Update dependencies list Java SDK README Signed-off-by: Danilo Del Busso --- .../java/autogen/xen-api/src/main/resources/README.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/resources/README.txt b/ocaml/sdk-gen/java/autogen/xen-api/src/main/resources/README.txt index ea1bf8c490e..632b8af5728 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/resources/README.txt +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/resources/README.txt @@ -39,5 +39,6 @@ https://discussions.citrix.com/forum/101-hypervisor-formerly-xenserver/ Dependencies ------------ -XenServerJava is dependent upon Apache XML-RPC by the Apache Software Foundation, -licensed under the Apache Software License 2.0. +XenServerJava is dependent upon: +- The jackson-databind (https://github.com/FasterXML/jackson-databind) package by the Jackson Project (https://github.com/FasterXML/jackson), licensed under the Apache Software License 2.0. +- The Apache HttpClient (https://hc.apache.org/httpcomponents-client/) package by the Apache Software Foundation (https://www.apache.org/), licensed under the Apache Software License 2.0. \ No newline at end of file From 166fa826426b279aad60ca58d10f0d8683cf96c9 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 16 Nov 2023 10:14:31 +0000 Subject: [PATCH 007/149] CP-45888: Fix formatting in Java's `main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 6a77a157b97..b0f3538978c 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -691,8 +691,8 @@ let gen_method_error_throw file name error = List.iter (fun i -> fprintf file - " String p%i = errorData.length > %i ? \ - errorData[%i] : \"\";\n" + " String p%i = errorData.length > %i ? errorData[%i] : \ + \"\";\n" i i i ) (range (List.length error.err_params)) ; @@ -706,10 +706,10 @@ let gen_types_class folder = print_license file ; fprintf file "package com.xensource.xenapi;\n\n\ - import java.util.Map;\n\n\ - import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;\n\ - import com.fasterxml.jackson.annotation.JsonProperty;\n\ - import java.io.IOException;\n\n\ + import java.util.Map;\n\n\ + import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;\n\ + import com.fasterxml.jackson.annotation.JsonProperty;\n\ + import java.io.IOException;\n\n\ /**\n\ \ * This class holds enum types and exceptions.\n\ \ */\n\ @@ -784,14 +784,14 @@ let gen_types_class folder = \ * returned an invalid response, throws a BadServerResponse. \ Otherwise, returns the server response as passed in.\n\ \ */\n\ - \ public static void checkError(JsonRpcResponseError response) throws XenAPIException, \ - BadServerResponse\n\ + \ public static void checkError(JsonRpcResponseError response) throws \ + XenAPIException, BadServerResponse\n\ \ {\n\ - \ var errorData = response.data; - \ if(errorData.length == 0){ - \ throw new BadServerResponse(response); - \ } - \ var errorName = errorData[0];\n\n" ; + \ var errorData = response.data;\n\ + \ if(errorData.length == 0){\n\ + \ throw new BadServerResponse(response);\n\ + \ }\n\ + \ var errorName = errorData[0];\n\n" ; Hashtbl.iter (gen_method_error_throw file) Datamodel.errors ; From c33b98166d63efc7f483a5352bef75e706abfe33 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 16 Nov 2023 15:19:35 +0000 Subject: [PATCH 008/149] Add missing `mli` files under `sdk-gen/` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/c/gen_c_binding.mli | 1 + ocaml/sdk-gen/c/helper.mli | 13 ++ ocaml/sdk-gen/common/CommonFunctions.mli | 130 ++++++++++++++++++ ocaml/sdk-gen/common/dune | 1 + ocaml/sdk-gen/common/license.mli | 2 + ocaml/sdk-gen/csharp/friendly_error_names.mli | 1 + ocaml/sdk-gen/csharp/gen_csharp_binding.mli | 1 + ocaml/sdk-gen/java/main.mli | 1 + ocaml/sdk-gen/powershell/common_functions.ml | 1 - ocaml/sdk-gen/powershell/common_functions.mli | 98 +++++++++++++ .../powershell/gen_powershell_binding.ml | 2 - .../powershell/gen_powershell_binding.mli | 1 + 12 files changed, 249 insertions(+), 3 deletions(-) create mode 100644 ocaml/sdk-gen/c/gen_c_binding.mli create mode 100644 ocaml/sdk-gen/c/helper.mli create mode 100644 ocaml/sdk-gen/common/CommonFunctions.mli create mode 100644 ocaml/sdk-gen/common/license.mli create mode 100644 ocaml/sdk-gen/csharp/friendly_error_names.mli create mode 100644 ocaml/sdk-gen/csharp/gen_csharp_binding.mli create mode 100644 ocaml/sdk-gen/java/main.mli create mode 100644 ocaml/sdk-gen/powershell/common_functions.mli create mode 100644 ocaml/sdk-gen/powershell/gen_powershell_binding.mli diff --git a/ocaml/sdk-gen/c/gen_c_binding.mli b/ocaml/sdk-gen/c/gen_c_binding.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/c/gen_c_binding.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) diff --git a/ocaml/sdk-gen/c/helper.mli b/ocaml/sdk-gen/c/helper.mli new file mode 100644 index 00000000000..1d8b90de096 --- /dev/null +++ b/ocaml/sdk-gen/c/helper.mli @@ -0,0 +1,13 @@ +val formatted_wrap : Format.formatter -> string -> unit +(** Recursively formats the input string + based on spaces and newlines, ensuring proper indentation. + + @param formatter The formatter to output the formatted string. + @param s The input string to be formatted. *) + +val comment : bool -> ?indent:int -> string -> string +(** Formats a comment block with optional indentation. + @param indent Optional indentation level. + @param doc Include an extra '*' for documentation. + @param s String content of the comment. + @return Formatted comment block. *) diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli new file mode 100644 index 00000000000..18a52821189 --- /dev/null +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -0,0 +1,130 @@ +(** Exception for unknown wire protocol. *) +exception Unknown_wire_protocol + +(** Type representing supported protocols. *) +type wireProtocol = XmlRpc | JsonRpc + +val get_release_branding : string -> string +(** [get_release_branding codename] Gets the branding for a release codename. + @param codename Release codename to lookup. + @return Branding for the release codename, or the original codename if not found. *) + +val is_setter : Datamodel_types.message -> bool +(** [is_setter message] Checks if a message is a setter based on its name. + @param message Message to check. + @return [true] if the message is a setter, [false] otherwise. *) + +val finally : (unit -> 'a) -> always:(unit -> unit) -> 'a +(** [finally f ~always] Executes [f] and then [always] regardless of success or failure. + @param f Function to execute. + @param always Function to execute always. *) + +val is_getter : Datamodel_types.message -> bool +(** [is_getter message] Checks if a message is a getter based on its name. + @param message Message to check. + @return [true] if the message is a getter, [false] otherwise. *) + +val get_deprecated_info_message : Datamodel_types.message -> string +(** [get_deprecated_info_message message] Returns a deprecated information message + based on the internal_deprecated_since version in the input message. + @param message Message containing version information. + @return Deprecated information message or an empty string if not deprecated. *) + +val is_adder : Datamodel_types.message -> bool +(** [is_adder message] Checks if a message is an adder based on its name. + @param message Message to check. + @return [true] if the message is an adder, [false] otherwise. *) + +val get_published_info_class : Datamodel_types.obj -> string +(** [get_published_info_class cls] Returns information about the first publication + of a class based on its lifecycle transitions. + @param cls Class to retrieve publication information for. + @return Information string with the first published version. *) + +val is_method_static : Datamodel_types.message -> bool +(** [is_method_static message] Checks if a method is static based on its parameters. + @param message Message to check. + @return [true] if the method is static, [false] otherwise. *) + +val escape_xml : string -> string +(** [escape_xml s] Escapes XML special characters in a string. + @param s String to escape. + @return String with XML special characters escaped. *) + +val get_published_info_message : + Datamodel_types.message -> Datamodel_types.obj -> string +(** Gets information about the publication status of a message. + @param message - Message to check. + @param cls - Class containing the message. + @return Information about the publication status. *) + +val is_remover : Datamodel_types.message -> bool +(** [is_remover message] Checks if a message is a remover based on its name. + @param message Message to check. + @return [true] if the message is a remover, [false] otherwise. *) + +val gen_param_groups : + Datamodel_types.message + -> Datamodel_types.param list + -> Datamodel_types.param list list +(** Generates parameter groups based on a message and its parameters. + @param message - Message containing the parameters. + @param params - List of parameters. + @return List of parameter groups. *) + +val get_published_info_param : + Datamodel_types.message -> Datamodel_types.param -> string +(** Gets information about the publication status of a parameter. + @param message - Message containing the parameter. + @param param - Parameter to check. + @return Information about the publication status. *) + +val get_published_info_field : + Datamodel_types.field -> Datamodel_types.obj -> string +(** Gets information about the publication status of a field within a class. + @param field - Field to check. + @param cls - Class containing the field. + @return Information about the publication status. *) + +val string_of_file : string -> string +(** [string_of_file filename] Reads the content of a file into a string. + @param filename Name of the file to read. + @return String containing the file content. *) + +val is_constructor : Datamodel_types.message -> bool +(** [is_constructor message] Checks if a message is a constructor. + @param message Message to check. + @return [true] if the message is a constructor, [false] otherwise. *) + +val with_output : string -> (out_channel -> 'a) -> 'a +(** [with_output filename f] Opens a file for writing and executes a function with the output channel. + @param filename Name of the file to open. + @param f Function to execute with the output channel. *) + +val is_destructor : Datamodel_types.message -> bool +(** [is_destructor message] Checks if a message is a destructor. + @param message Message to check. + @return [true] if the message is a destructor, [false] otherwise. *) + +val is_real_constructor : Datamodel_types.message -> bool +(** [is_real_constructor message] Checks if a message is a real constructor. + @param message Message to check. + @return [true] if the message is a real constructor, [false] otherwise. *) + +val render_file : string * string -> Mustache.Json.t -> string -> string -> unit +(** [render_file (infile, outfile) json templates_dir dest_dir] Renders a file using a JSON data model and templates. + @param infile Input file name. + @param outfile Output file name. + @param json JSON data model. + @param templates_dir Directory containing templates. + @param dest_dir Directory for the rendered output. *) + +val json_releases : + [> `O of + ( string + * [> `A of + [> `O of (string * [> `Float of float | `String of string]) list] list + | `Float of float ] + ) + list ] +(** JSON structure representing release information. *) diff --git a/ocaml/sdk-gen/common/dune b/ocaml/sdk-gen/common/dune index 2278becc3e0..7cda0194598 100644 --- a/ocaml/sdk-gen/common/dune +++ b/ocaml/sdk-gen/common/dune @@ -6,5 +6,6 @@ xapi-datamodel mustache ) + (modules_without_implementation license) ) diff --git a/ocaml/sdk-gen/common/license.mli b/ocaml/sdk-gen/common/license.mli new file mode 100644 index 00000000000..6ab5ae4cc08 --- /dev/null +++ b/ocaml/sdk-gen/common/license.mli @@ -0,0 +1,2 @@ +(* Conent of BSD 2 License *) +val bsd_two_clause : string diff --git a/ocaml/sdk-gen/csharp/friendly_error_names.mli b/ocaml/sdk-gen/csharp/friendly_error_names.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/csharp/friendly_error_names.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.mli b/ocaml/sdk-gen/csharp/gen_csharp_binding.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) diff --git a/ocaml/sdk-gen/java/main.mli b/ocaml/sdk-gen/java/main.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/java/main.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index d81f9f687db..d3b69b7d24a 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -197,7 +197,6 @@ and cut_msg_name message_name fn_type = else message_name -(* True if an object has a uuid (and therefore should have a get_by_uuid message *) and has_uuid x = let all_fields = DU.fields_of_obj x in List.filter (fun fld -> fld.full_name = ["uuid"]) all_fields <> [] diff --git a/ocaml/sdk-gen/powershell/common_functions.mli b/ocaml/sdk-gen/powershell/common_functions.mli new file mode 100644 index 00000000000..706264469f4 --- /dev/null +++ b/ocaml/sdk-gen/powershell/common_functions.mli @@ -0,0 +1,98 @@ +val get_http_action_stem : string -> string +(** Gets the HTTP action stem based on the name. + @param name - Name to analyze. + @return HTTP action stem. *) + +val get_http_action_verb : string -> Datamodel.http_meth -> string +(** Gets the HTTP action verb based on the name and method. + @param name - Name to analyze. + @param meth - HTTP method. + @return HTTP action verb. *) + +val get_common_verb_category : string -> string +(** Gets the common verb category based on the HTTP action verb. + @param verb - HTTP action verb. + @return Common verb category. *) + +val pascal_case_ : string -> string +(** Recursively converts a string to PascalCase. + @param s - String to convert. + @return PascalCase formatted string. *) + +val qualified_class_name : string -> string +(** Gets the qualified class name by prepending "XenAPI." to the exposed class name. + @param classname - Class name. + @return Qualified class name. *) + +val ocaml_class_to_csharp_local_var : string -> string +(** Converts an OCaml class name to a C# local variable. + @param classname - OCaml class name. + @return C# local variable name. *) + +val ocaml_class_to_csharp_property : string -> string +(** Converts an OCaml class name to a C# property name. + @param classname - OCaml class name. + @return C# property name. *) + +val ocaml_class_to_csharp_class : string -> string +(** Converts an OCaml class name to a C# class name. + @param classname - OCaml class name. + @return C# class name. *) + +val is_invoke : Datamodel_types.message -> bool +(** Checks if a message is an invoke operation. + @param message - Message to check. + @return true if it's an invoke operation, false otherwise. *) + +val has_uuid : Datamodel_types.obj -> bool +(** Checks if an object has a UUID field, and therefore should have a get_by_uuid message. + @param x - Object to check. + @return true if the object has a UUID field, false otherwise. *) + +val has_name : Datamodel_types.obj -> bool +(** Checks if an object has a name field. + @param x - Object to check. + @return true if the object has a name field, false otherwise. *) + +val full_name : Datamodel_types.field -> string +(** Gets the full name of a field as a single string with underscores escaped. + @param field - Field to extract the full name from. + @return Full name with underscores escaped. *) + +val obj_internal_type : Datamodel_types.ty -> string +(** Gets the internal type representation of an object. + @param x - Object to determine the internal type for. + @return Internal type representation as a string. *) + +val ocaml_field_to_csharp_property : Datamodel_types.field -> string +(** Converts an OCaml field to its corresponding C# property name. + @param field - OCaml field. + @return C# property name. *) + +val exposed_type : Datamodel_types.ty -> string +(** Converts an exposed type from OCaml to its corresponding C# type. + @param ty - OCaml type. + @return Corresponding C# type. *) + +val pascal_case : string -> string +(** Converts a string to PascalCase. + @param s - Input string. + @return PascalCase formatted string. *) + +val exposed_class_name : string -> string +(** Converts an OCaml class name to a corresponding exposed class name in C#. + @param classname - OCaml class name. + @return Exposed class name in C#. *) + +val cut_msg_name : string -> string -> string +(** Extracts the base name from an OCaml message name by removing the specified prefix. + Some adders/removers are just prefixed by Add or RemoveFrom + and some are prefixed by AddTo or RemoveFrom + @param message_name - OCaml message name. + @param fn_type - Prefix to remove ("Add" or "Remove"). + @return Base name after removing the specified prefix. *) + +val lower_and_underscore_first : string -> string +(** Converts a string to lowercase and adds an underscore at the beginning. + @param s - Input string. + @return String in lowercase with an underscore at the beginning. *) diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 2e1fb55e5fb..406a8ddbfc1 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -19,8 +19,6 @@ end) let destdir = "autogen/src" -let templdir = "templates" - type cmdlet = {filename: string; content: string} let api = diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.mli b/ocaml/sdk-gen/powershell/gen_powershell_binding.mli new file mode 100644 index 00000000000..c8b99626f9c --- /dev/null +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.mli @@ -0,0 +1 @@ +(* Empty .mli to ensure unused functions are picked up during check*) From 909aeaaee561bb9380a8fa1559b9f4d900ec0999 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 16 Nov 2023 15:36:47 +0000 Subject: [PATCH 009/149] Remove all unused SDK generation functions As reported by `make check` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/c/gen_c_binding.ml | 5 --- ocaml/sdk-gen/common/CommonFunctions.ml | 3 -- ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 38 ------------------ ocaml/sdk-gen/powershell/common_functions.ml | 32 --------------- .../powershell/gen_powershell_binding.ml | 40 ------------------- 5 files changed, 118 deletions(-) diff --git a/ocaml/sdk-gen/c/gen_c_binding.ml b/ocaml/sdk-gen/c/gen_c_binding.ml index 0c84af4ac93..7121ea70f34 100644 --- a/ocaml/sdk-gen/c/gen_c_binding.ml +++ b/ocaml/sdk-gen/c/gen_c_binding.ml @@ -1412,15 +1412,10 @@ and internal_decl_filename name = and impl_filename name = sprintf "xen_%s.c" (String.lowercase_ascii name) -and internal_impl_filename name = - sprintf "xen_%s_internal.c" (String.lowercase_ascii name) - and protector classname = sprintf "XEN_%s_H" (String.uppercase_ascii classname) and typename classname = sprintf "xen_%s" (String.lowercase_ascii classname) -and variablename classname = sprintf "%s" (String.lowercase_ascii classname) - and record_typename classname = sprintf "%s_record" (typename classname) and record_opt_typename classname = sprintf "%s_record_opt" (typename classname) diff --git a/ocaml/sdk-gen/common/CommonFunctions.ml b/ocaml/sdk-gen/common/CommonFunctions.ml index 2d046933bf1..cd262792b57 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.ml +++ b/ocaml/sdk-gen/common/CommonFunctions.ml @@ -40,9 +40,6 @@ let with_output filename f = let io = open_out filename in finally (fun () -> f io) ~always:(fun () -> close_out io) -let joined sep f l = - l |> List.map f |> List.filter (fun x -> x <> "") |> String.concat sep - let escape_xml s = s |> Astring.String.cuts ~sep:"<" ~empty:true diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index 2d7f254fad9..db12da58b29 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -137,10 +137,6 @@ and gen_relations () = Hashtbl.iter (gen_relations_by_type out_chan) relations ; print "\n return relations;\n }\n }\n}\n" -and string_ends str en = - let len = String.length en in - String.sub str (String.length str - len) len = en - and process_relations ((oneClass, oneField), (manyClass, manyField)) = let value = try (manyField, oneClass, oneField) :: Hashtbl.find relations manyClass @@ -451,16 +447,6 @@ and get_constructor_body' content elements = | Namespace (_, c) :: others -> get_constructor_body' (c @ others) elements -and gen_constructor_line out_chan content = - let print format = fprintf out_chan format in - - match content with - | Field fr -> - print " %s = %s;\n" (full_name fr) - (convert_from_proxy ("proxy." ^ full_name fr) fr.ty) - | Namespace (_, c) -> - List.iter (gen_constructor_line out_chan) c - and gen_hashtable_constructor_line out_chan content = let print format = fprintf out_chan format in @@ -560,9 +546,6 @@ and gen_exposed_method cls msg curParams = in sync ^ async -and returns_xenobject msg = - match msg.msg_result with Some (Record _, _) -> true | _ -> false - and get_params_doc msg classname params = let sessionDoc = "\n /// The session" @@ -688,18 +671,6 @@ and gen_save_changes_to_field out_chan exposed_class_name fr = \ }\n" equality exposed_class_name full_name_fr full_name_fr -and ctor_call classname = - let fields = - Datamodel_utils.fields_of_obj (Dm_api.get_obj_by_name api ~objname:classname) - in - let fields2 = - List.filter - (function {DT.qualifier= DT.StaticRO | DT.RW; _} -> true | _ -> false) - fields - in - let args = List.map (fun fr -> "p." ^ full_name fr) fields2 in - String.concat ", " ("session.opaque_ref" :: args) - and gen_exposed_field out_chan cls content = match content with | Field fr -> @@ -868,15 +839,6 @@ and gen_enum' name contents = ("enum", `String name); ("enum_members", `A (List.map enum_member members)) ] -and has_unknown_entry contents = - let rec f = function - | x :: xs -> - if String.lowercase_ascii (fst x) = "unknown" then true else f xs - | [] -> - false - in - f contents - (* ------------------- category: maps *) and gen_maps () = let out_chan = open_out (Filename.concat destdir "Maps.cs") in diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index d3b69b7d24a..a2832c14494 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -59,9 +59,6 @@ and ocaml_class_to_csharp_local_var classname = else String.lowercase_ascii (exposed_class_name classname) -and ocaml_field_to_csharp_local_var field = - String.lowercase_ascii (full_name field) - and ocaml_field_to_csharp_property field = ocaml_class_to_csharp_property (full_name field) @@ -86,39 +83,10 @@ and exposed_class_name classname = and qualified_class_name classname = "XenAPI." ^ exposed_class_name classname -and type_default ty = - match ty with - | Int -> - "" - | SecretString | String -> - "" - | Float -> - "" - | Bool -> - "" - | Enum _ -> - "" - | Record _ -> - "" - | Ref _ -> - "" - | Map (_, _) -> - " = new Hashtable()" - | Set String -> - " = new string[0]" - | _ -> - sprintf " = new %s()" (exposed_type ty) - and escaped = function "params" -> "paramz" | s -> s and full_name field = escaped (String.concat "_" field.full_name) -and exposed_type_opt = function - | Some (typ, _) -> - exposed_type typ - | None -> - "void" - and exposed_type = function | SecretString | String -> "string" diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 406a8ddbfc1..96d0782bf67 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -552,15 +552,6 @@ and print_methods_constructor message obj classname = (gen_shouldprocess "New" message classname) (gen_csharp_api_call message classname "New" "passthru") -and create_param_parse param paramName = - match param.param_type with - | Ref _ -> - sprintf "\n string %s = %s.opaque_ref;\n" - (String.lowercase_ascii param.param_name) - paramName - | _ -> - "" - and gen_make_record obj classname = sprintf "\n\ @@ -1145,37 +1136,6 @@ and print_cmdlet_methods_dynamic classname messages enum commonVerb = enum (cut_message_name hd) (cut_message_name hd) localVar (print_cmdlet_methods_dynamic classname tl enum commonVerb) -and print_async_param_getter classname asyncMessages = - let properties = - List.map - (fun x -> - sprintf " case Xen%sProperty.%s:" - (ocaml_class_to_csharp_class classname) - x - ) - asyncMessages - in - match asyncMessages with - | [] -> - "" - | _ -> - sprintf - "\n\ - \ protected override bool GenerateAsyncParam\n\ - \ {\n\ - \ get\n\ - \ {\n\ - \ switch (XenProperty)\n\ - \ {\n\ - %s\n\ - \ return true;\n\ - \ default:\n\ - \ return false;\n\ - \ }\n\ - \ }\n\ - \ }\n" - (String.concat "\n" properties) - (**************************************) (* Common to more than one generators *) (**************************************) From 64525404d26dc74d69b72f9f8910dd759d7d9f31 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 16 Nov 2023 15:40:13 +0000 Subject: [PATCH 010/149] Update number of `mli` files in quality gate script Signed-off-by: Danilo Del Busso --- quality-gate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quality-gate.sh b/quality-gate.sh index 224e852aa32..77238f4ab93 100755 --- a/quality-gate.sh +++ b/quality-gate.sh @@ -25,7 +25,7 @@ verify-cert () { } mli-files () { - N=530 + N=522 # do not count ml files from the tests in ocaml/{tests/perftest/quicktest} MLIS=$(git ls-files -- '**/*.mli' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) MLS=$(git ls-files -- '**/*.ml' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) From 16e84547dc16bf3a8a360dc72e2f7d4547b700b3 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 20 Nov 2023 09:29:50 +0000 Subject: [PATCH 011/149] Use `Mustache.Json.t` as type for `json_releases` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/common/CommonFunctions.mli | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli index 18a52821189..fd2ba766568 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.mli +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -119,12 +119,5 @@ val render_file : string * string -> Mustache.Json.t -> string -> string -> unit @param templates_dir Directory containing templates. @param dest_dir Directory for the rendered output. *) -val json_releases : - [> `O of - ( string - * [> `A of - [> `O of (string * [> `Float of float | `String of string]) list] list - | `Float of float ] - ) - list ] +val json_releases : Mustache.Json.t (** JSON structure representing release information. *) From 3c9b98b1a670a5ea00f82d21906a3bc55eff4952 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 20 Nov 2023 09:30:25 +0000 Subject: [PATCH 012/149] Remove use of custom `finally` in `sdk-gen` Instead use `Fun.protect` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/c/gen_c_binding.ml | 18 ++++++++++-------- ocaml/sdk-gen/common/CommonFunctions.ml | 17 +++++------------ ocaml/sdk-gen/common/CommonFunctions.mli | 5 ----- ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 8 +++++--- 4 files changed, 20 insertions(+), 28 deletions(-) diff --git a/ocaml/sdk-gen/c/gen_c_binding.ml b/ocaml/sdk-gen/c/gen_c_binding.ml index 7121ea70f34..5203b86f2e0 100644 --- a/ocaml/sdk-gen/c/gen_c_binding.ml +++ b/ocaml/sdk-gen/c/gen_c_binding.ml @@ -138,14 +138,16 @@ let rec main () = and gen_class f g clas targetdir = let out_chan = open_out (Filename.concat targetdir (g clas.name)) in - finally (fun () -> f clas out_chan) ~always:(fun () -> close_out out_chan) + Fun.protect (fun () -> f clas out_chan) ~finally:(fun () -> close_out out_chan) and gen_enum f g targetdir = function | Enum (name, _) as x -> if not (List.mem name !all_headers) then all_headers := name :: !all_headers ; let out_chan = open_out (Filename.concat targetdir (g name)) in - finally (fun () -> f x out_chan) ~always:(fun () -> close_out out_chan) + Fun.protect + (fun () -> f x out_chan) + ~finally:(fun () -> close_out out_chan) | _ -> assert false @@ -155,9 +157,9 @@ and gen_map f g targetdir = function if not (List.mem name !all_headers) then all_headers := name :: !all_headers ; let out_chan = open_out (Filename.concat targetdir (g name)) in - finally + Fun.protect (fun () -> f name l r out_chan) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) | _ -> assert false @@ -972,14 +974,14 @@ and gen_failure_h () = let out_chan = open_out (Filename.concat destdir "include/xen/api/xen_api_failure.h") in - finally + Fun.protect (fun () -> print_h_header out_chan protect ; gen_failure_enum out_chan ; gen_failure_funcs out_chan ; print_h_footer out_chan ) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) and gen_failure_enum out_chan = let print format = fprintf out_chan format in @@ -1029,7 +1031,7 @@ and gen_failure_funcs out_chan = and gen_failure_c () = let out_chan = open_out (Filename.concat destdir "src/xen_api_failure.c") in let print format = fprintf out_chan format in - finally + Fun.protect (fun () -> print "%s\n\n\ @@ -1055,7 +1057,7 @@ and gen_failure_c () = Licence.bsd_two_clause (String.concat ",\n " (failure_lookup_entries ())) ) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) and failure_lookup_entries () = List.sort String.compare diff --git a/ocaml/sdk-gen/common/CommonFunctions.ml b/ocaml/sdk-gen/common/CommonFunctions.ml index cd262792b57..fe89dd9600e 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.ml +++ b/ocaml/sdk-gen/common/CommonFunctions.ml @@ -11,13 +11,6 @@ type wireProtocol = XmlRpc | JsonRpc type rpm_version = {major: int; minor: int; micro: int} -let finally f ~(always : unit -> unit) = - match f () with - | result -> - always () ; result - | exception e -> - always () ; raise e - let parse_to_rpm_version_option inputStr = try Scanf.sscanf inputStr "%d.%d.%d" (fun x y z -> @@ -27,18 +20,18 @@ let parse_to_rpm_version_option inputStr = let string_of_file filename = let in_channel = open_in filename in - finally + Fun.protect (fun () -> let rec read_lines acc = try read_lines (input_line in_channel :: acc) with End_of_file -> acc in read_lines [] |> List.rev |> String.concat "\n" ) - ~always:(fun () -> close_in in_channel) + ~finally:(fun () -> close_in in_channel) let with_output filename f = let io = open_out filename in - finally (fun () -> f io) ~always:(fun () -> close_out io) + Fun.protect (fun () -> f io) ~finally:(fun () -> close_out io) let escape_xml s = s @@ -269,9 +262,9 @@ and render_template template_file json output_file = let templ = string_of_file template_file |> Mustache.of_string in let rendered = Mustache.render templ json in let out_chan = open_out output_file in - finally + Fun.protect (fun () -> output_string out_chan rendered) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) let render_file (infile, outfile) json templates_dir dest_dir = let input_path = Filename.concat templates_dir infile in diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli index fd2ba766568..d114f6ea69a 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.mli +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -14,11 +14,6 @@ val is_setter : Datamodel_types.message -> bool @param message Message to check. @return [true] if the message is a setter, [false] otherwise. *) -val finally : (unit -> 'a) -> always:(unit -> unit) -> 'a -(** [finally f ~always] Executes [f] and then [always] regardless of success or failure. - @param f Function to execute. - @param always Function to execute always. *) - val is_getter : Datamodel_types.message -> bool (** [is_getter message] Checks if a message is a getter based on its name. @param message Message to check. diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index db12da58b29..0c70b85c6d6 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -240,9 +240,9 @@ and gen_class_file cls = let out_chan = open_out (Filename.concat destdir (exposed_class_name cls.name) ^ ".cs") in - finally + Fun.protect (fun () -> gen_class out_chan cls) - ~always:(fun () -> close_out out_chan) + ~finally:(fun () -> close_out out_chan) and gen_class out_chan cls = let print format = fprintf out_chan format in @@ -842,7 +842,9 @@ and gen_enum' name contents = (* ------------------- category: maps *) and gen_maps () = let out_chan = open_out (Filename.concat destdir "Maps.cs") in - finally (fun () -> gen_maps' out_chan) ~always:(fun () -> close_out out_chan) + Fun.protect + (fun () -> gen_maps' out_chan) + ~finally:(fun () -> close_out out_chan) and gen_maps' out_chan = let print format = fprintf out_chan format in From 3d9dc50b895af9bb305d26f8f6e00f622ffe44da Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 20 Nov 2023 11:34:12 +0000 Subject: [PATCH 013/149] Replace usage of escaped strings with `{||}` in Java's `main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 213 +++++++++++++++++++------------------ 1 file changed, 108 insertions(+), 105 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index b0f3538978c..3fddff31b96 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -520,16 +520,18 @@ let gen_class cls folder = let publishInfo = get_published_info_class cls in print_license file ; fprintf file - "package com.xensource.xenapi;\n\n\ - import com.fasterxml.jackson.annotation.JsonProperty;\n\ - import com.fasterxml.jackson.core.JsonProcessingException;\n\ - import com.fasterxml.jackson.core.type.TypeReference;\n\ - import com.xensource.xenapi.Types.BadServerResponse;\n\ - import com.xensource.xenapi.Types.XenAPIException;\n\n\ - import java.io.PrintWriter;\n\ - import java.io.StringWriter;\n\ - import java.util.*;\n\ - import java.io.IOException;\n\n" ; + {|package com.xensource.xenapi; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.xensource.xenapi.Types.BadServerResponse; +import com.xensource.xenapi.Types.XenAPIException; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.util.*; +import java.io.IOException; + +|} ; fprintf file "/**\n" ; fprintf file " * %s\n" cls.description ; if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; @@ -684,122 +686,123 @@ let gen_method_error_throw file name error = ) in - fprintf file " if (errorName.equals(\"%s\"))\n" name ; - fprintf file " {\n" ; + fprintf file " if (errorName.equals(\"%s\")){\n" name ; (* Prepare the parameters to the Exception constructor *) List.iter (fun i -> fprintf file - " String p%i = errorData.length > %i ? errorData[%i] : \ - \"\";\n" + " String p%i = errorData.length > %i ? errorData[%i] : \"\";\n" i i i ) (range (List.length error.err_params)) ; - fprintf file " throw new Types.%s(%s);\n" class_name paramsStr ; - fprintf file " }\n" + fprintf file " throw new Types.%s(%s);\n" class_name paramsStr ; + fprintf file " }\n" let gen_types_class folder = let class_name = "Types" in let file = open_out (Filename.concat folder class_name ^ ".java") in print_license file ; fprintf file - "package com.xensource.xenapi;\n\n\ - import java.util.Map;\n\n\ - import com.fasterxml.jackson.annotation.JsonEnumDefaultValue;\n\ - import com.fasterxml.jackson.annotation.JsonProperty;\n\ - import java.io.IOException;\n\n\ - /**\n\ - \ * This class holds enum types and exceptions.\n\ - \ */\n\ - public class Types\n\ - {\n\ - \ /**\n\ - \ * Interface for all Record classes\n\ - \ */\n\ - \ public interface Record\n\ - \ {\n\ - \ /**\n\ - \ * Convert a Record to a Map\n\ - \ */\n\ - \ Map toMap();\n\ - \ }\n\n\ - \ /**\n\ - \ * Base class for all XenAPI Exceptions\n\ - \ */\n\ - \ public static class XenAPIException extends IOException {\n\ - \ public final String shortDescription;\n\ - \ public final String[] errorDescription;\n\n\ - \ XenAPIException(String shortDescription)\n\ - \ {\n\ - \ this.shortDescription = shortDescription;\n\ - \ this.errorDescription = null;\n\ - \ }\n\n\ - \ XenAPIException(String[] errorDescription)\n\ - \ {\n\ - \ this.errorDescription = errorDescription;\n\n\ - \ if (errorDescription.length > 0)\n\ - \ {\n\ - \ shortDescription = errorDescription[0];\n\ - \ } else\n\ - \ {\n\ - \ shortDescription = \"\";\n\ - \ }\n\ - \ }\n\n\ - \ public String toString()\n\ - \ {\n\ - \ if (errorDescription == null)\n\ - \ {\n\ - \ return shortDescription;\n\ - \ } else if (errorDescription.length == 0)\n\ - \ {\n\ - \ return \"\";\n\ - \ }\n\ - \ StringBuilder sb = new StringBuilder();\n\ - \ for (int i = 0; i < errorDescription.length - 1; i++)\n\ - \ {\n\ - \ sb.append(errorDescription[i]);\n\ - \ }\n\ - \ sb.append(errorDescription[errorDescription.length - 1]);\n\n\ - \ return sb.toString();\n\ - \ }\n\ - \ }\n\ - \ /**\n\ - \ * Thrown if the response from the server contains an invalid status.\n\ - \ */\n\ - \ public static class BadServerResponse extends XenAPIException\n\ - \ {\n\ - \ public BadServerResponse(JsonRpcResponseError responseError)\n\ - \ {\n\ - \ super(String.valueOf(responseError));\n\ - \ }\n\ - \ }\n\n\ - \ " ; + {|package com.xensource.xenapi; +import java.util.Map; +import com.fasterxml.jackson.annotation.JsonEnumDefaultValue; +import com.fasterxml.jackson.annotation.JsonProperty; +import java.io.IOException; + +/** + * This class holds enum types and exceptions. + */ +public class Types +{ + /** + * Interface for all Record classes + */ + public interface Record + { + /** + * Convert a Record to a Map + */ + Map toMap(); + } + /** + * Base class for all XenAPI Exceptions + */ + public static class XenAPIException extends IOException { + public final String shortDescription; + public final String[] errorDescription; + XenAPIException(String shortDescription) + { + this.shortDescription = shortDescription; + this.errorDescription = null; + } + XenAPIException(String[] errorDescription) + { + this.errorDescription = errorDescription; + if (errorDescription.length > 0) + { + shortDescription = errorDescription[0]; + } else + { + shortDescription = ""; + } + } + public String toString() + { + if (errorDescription == null) + { + return shortDescription; + } else if (errorDescription.length == 0) + { + return ""; + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < errorDescription.length - 1; i++) + { + sb.append(errorDescription[i]); + } + sb.append(errorDescription[errorDescription.length - 1]); + return sb.toString(); + } + } + /** + * Thrown if the response from the server contains an invalid status. + */ + public static class BadServerResponse extends XenAPIException + { + public BadServerResponse(JsonRpcResponseError responseError) + { + super(String.valueOf(responseError)); + } + } +|} ; fprintf file - " /**\n\ - \ * Checks the provided server response was successful. If the call \ - failed, throws a XenAPIException. If the server\n\ - \ * returned an invalid response, throws a BadServerResponse. \ - Otherwise, returns the server response as passed in.\n\ - \ */\n\ - \ public static void checkError(JsonRpcResponseError response) throws \ - XenAPIException, BadServerResponse\n\ - \ {\n\ - \ var errorData = response.data;\n\ - \ if(errorData.length == 0){\n\ - \ throw new BadServerResponse(response);\n\ - \ }\n\ - \ var errorName = errorData[0];\n\n" ; + {| /** + * Checks the provided server response was successful. If the call + * failed, throws a XenAPIException. If the server + * returned an invalid response, throws a BadServerResponse. + * Otherwise, returns the server response as passed in. + */ + public static void checkError(JsonRpcResponseError response) throws XenAPIException, BadServerResponse + { + var errorData = response.data; + if(errorData.length == 0){ + throw new BadServerResponse(response); + } + var errorName = errorData[0]; +|} ; Hashtbl.iter (gen_method_error_throw file) Datamodel.errors ; fprintf file - "\n\ - \ // An unknown error occurred\n\ - \ throw new Types.XenAPIException(errorData);\n\ - \ }\n\n" ; + {| + // An unknown error occurred + throw new Types.XenAPIException(errorData); +} + +|} ; gen_enums file ; fprintf file "\n" ; From 1224222420015470c7bfe1c54e0f2f7ec08656a9 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 20 Nov 2023 11:39:36 +0000 Subject: [PATCH 014/149] Replace mentions of `java.net.http.HttpClient` in `JsonRpcClient.java` Signed-off-by: Danilo Del Busso --- .../src/main/java/com/xensource/xenapi/JsonRpcClient.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index 8747822672c..0dad4b7dc0a 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -44,7 +44,6 @@ import java.io.IOException; import java.net.URL; -import java.net.http.HttpClient; import java.text.SimpleDateFormat; import java.util.concurrent.TimeUnit; @@ -61,7 +60,7 @@ * By default, the timeout for requests is set to 10 minutes (600 seconds). The default timeout for connecting to the * JSON-RPC backend is set to 5 seconds. * - * @see HttpClient HttpClient is used to make requests and connect to the backend + * @see CloseableHttpClient CloseableHttpClient is used to make requests and connect to the backend * @see ObjectMapper ObjectMapper is used to marshall requests and responses */ public class JsonRpcClient { @@ -120,12 +119,12 @@ public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTi } /** - * Initialize a JsonRpcClient using a custom HttpClient instance. + * Initialize a JsonRpcClient using a custom CloseableHttpClient instance. * * @param client the custom HttpClient to use for all requests * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. * @param requestTimeout the timeout value for requests. - * @see HttpClient + * @see CloseableHttpClient CloseableHttpClient the client that will be used for dispatching requests * @see JsonRpcClient JsonRpcClient for more info on using this class */ public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requestTimeout) { From 3f9ed3956f4b772e616d72ea907f7a4e3e8fc846 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 27 Nov 2023 08:44:05 +0000 Subject: [PATCH 015/149] CP-45888: Do not expose `JsonProcessingException` to SDK users Exception is already subclass of stdlib exception `IOException` and there's no need to expose it explicitly Signed-off-by: Danilo Del Busso --- .../main/java/com/xensource/xenapi/Connection.java | 11 ++++------- ocaml/sdk-gen/java/main.ml | 9 ++++----- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index f568c452894..fcf64b78485 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -29,7 +29,6 @@ package com.xensource.xenapi; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.xensource.xenapi.Types.BadServerResponse; import com.xensource.xenapi.Types.XenAPIException; @@ -203,10 +202,9 @@ public String getSessionReference() { * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records * @return The result of the call with the type specified under T. * @throws XenAPIException if the call failed. - * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON - * @throws IOException if an I/O error occurs when sending or receiving + * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. */ - public T dispatch(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws XenAPIException, JsonProcessingException, IOException { + public T dispatch(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws XenAPIException, IOException { var result = client.sendRequest(methodCall, methodParameters, responseTypeReference); if (result.error != null) { throw new XenAPIException(String.valueOf(result.error)); @@ -231,10 +229,9 @@ public T dispatch(String methodCall, Object[] methodParameters, TypeReferenc * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password * @param methodParameters the methodParameters of the method call * @throws XenAPIException if the call failed. - * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON - * @throws IOException if an I/O error occurs when sending or receiving + * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. */ - public void dispatch(String methodCall, Object[] methodParameters) throws XenAPIException, JsonProcessingException, IOException { + public void dispatch(String methodCall, Object[] methodParameters) throws XenAPIException, IOException { var typeReference = new TypeReference() { }; this.dispatch(methodCall, methodParameters, typeReference); diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 3fddff31b96..03d3ab73b8e 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -233,11 +233,11 @@ let gen_method file cls message params async_version = , "Thrown if the response from the server contains an invalid status." ) ; ("XenAPIException", "if the call failed.") - ; ( "JsonProcessingException" - , "if the request's payload or the response's payload cannot be written \ - or read as valid JSON." + ; ( "IOException" + , "if an I/O error occurs when sending or receiving, includes cases when \ + the request's payload or the response's payload cannot be written or \ + read as valid JSON." ) - ; ("IOException", "if an I/O error occurs when sending or receiving.") ] in let publishInfo = get_published_info_message message cls in @@ -522,7 +522,6 @@ let gen_class cls folder = fprintf file {|package com.xensource.xenapi; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.xensource.xenapi.Types.BadServerResponse; import com.xensource.xenapi.Types.XenAPIException; From 13c1a29b44a4cf992721d174cadcc1ce6b9f91d7 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 8 Jan 2024 15:36:45 +0000 Subject: [PATCH 016/149] CP-45888: Fix misc typos and comments Also remove a redundant `List.map` in `java/main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/c/helper.mli | 2 +- ocaml/sdk-gen/common/license.mli | 2 +- .../main/java/com/xensource/xenapi/Connection.java | 2 +- .../java/com/xensource/xenapi/JsonRpcClient.java | 2 +- ocaml/sdk-gen/java/main.ml | 3 +-- ocaml/sdk-gen/powershell/common_functions.ml | 2 +- ocaml/sdk-gen/powershell/common_functions.mli | 14 +++++++------- 7 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ocaml/sdk-gen/c/helper.mli b/ocaml/sdk-gen/c/helper.mli index 1d8b90de096..ea32b78f207 100644 --- a/ocaml/sdk-gen/c/helper.mli +++ b/ocaml/sdk-gen/c/helper.mli @@ -1,6 +1,6 @@ val formatted_wrap : Format.formatter -> string -> unit (** Recursively formats the input string - based on spaces and newlines, ensuring proper indentation. + based on spaces and new lines, ensuring proper indentation. @param formatter The formatter to output the formatted string. @param s The input string to be formatted. *) diff --git a/ocaml/sdk-gen/common/license.mli b/ocaml/sdk-gen/common/license.mli index 6ab5ae4cc08..092b79357a6 100644 --- a/ocaml/sdk-gen/common/license.mli +++ b/ocaml/sdk-gen/common/license.mli @@ -1,2 +1,2 @@ -(* Conent of BSD 2 License *) +(* Content of BSD 2 License *) val bsd_two_clause : string diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index fcf64b78485..6005982aaa6 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -199,7 +199,7 @@ public String getSessionReference() { * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password * @param methodParameters the methodParameters of the method call * @param responseTypeReference the type of the response, wrapped with a TypeReference - * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records + * @param The type of the response's payload. For instance, a map of opaque references to VM objects is expected when calling VM.get_all_records * @return The result of the call with the type specified under T. * @throws XenAPIException if the call failed. * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index 0dad4b7dc0a..7172e148f4c 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -143,7 +143,7 @@ public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requ * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password * @param methodParameters the parameters of the method call * @param responseTypeReference the type of the response, wrapped with a TypeReference - * @param The type of the response's payload. For instance, an array of VMs is expected when calling VM.get_all_records + * @param The type of the response's payload. For instance, a map of opaque references to VM objects is expected when calling VM.get_all_records * @return a JsonRpcResponse object. If its error field is empty, the response was successful. * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON * @throws IOException if an I/O error occurs when sending or receiving diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 03d3ab73b8e..b45f02c2fc8 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -336,8 +336,7 @@ let gen_method file cls message params async_version = get_method_params_for_xml message params in - output_string file - (String.concat ", " (List.map (fun s -> sprintf "%s" s) methodParamsList)) ; + output_string file (String.concat ", " methodParamsList) ; fprintf file "};\n" ; diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index a2832c14494..fd47311eba5 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -144,7 +144,7 @@ and is_invoke message = && (not (is_constructor message)) && not (is_destructor message) -(* Some adders/removers are just prefixed by Add or RemoveFrom +(* Some adders/removers are just prefixed by Add or Remove and some are prefixed by AddTo or RemoveFrom *) and cut_msg_name message_name fn_type = let name_len = String.length message_name in diff --git a/ocaml/sdk-gen/powershell/common_functions.mli b/ocaml/sdk-gen/powershell/common_functions.mli index 706264469f4..f947139bec0 100644 --- a/ocaml/sdk-gen/powershell/common_functions.mli +++ b/ocaml/sdk-gen/powershell/common_functions.mli @@ -86,13 +86,13 @@ val exposed_class_name : string -> string val cut_msg_name : string -> string -> string (** Extracts the base name from an OCaml message name by removing the specified prefix. - Some adders/removers are just prefixed by Add or RemoveFrom - and some are prefixed by AddTo or RemoveFrom - @param message_name - OCaml message name. - @param fn_type - Prefix to remove ("Add" or "Remove"). - @return Base name after removing the specified prefix. *) + Some adders/removers are just prefixed by Add or Remove and some are prefixed by + AddTo or RemoveFrom. + @param message_name - OCaml message name. + @param fn_type - Prefix to remove ("Add" or "Remove"). + @return Base name after removing the specified prefix. *) val lower_and_underscore_first : string -> string (** Converts a string to lowercase and adds an underscore at the beginning. - @param s - Input string. - @return String in lowercase with an underscore at the beginning. *) + @param s - Input string. + @return String in lowercase with an underscore at the beginning. *) From dbe7e11bd3e78ba4b763b44497c6f6c3b7d4b1e8 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 9 Jan 2024 15:18:11 +0000 Subject: [PATCH 017/149] Add missing function signature in `CommonFunctions.mli` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/common/CommonFunctions.mli | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli index d114f6ea69a..2b2312fafb7 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.mli +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -58,6 +58,13 @@ val is_remover : Datamodel_types.message -> bool @param message Message to check. @return [true] if the message is a remover, [false] otherwise. *) +val get_minimum_allowed_role : Datamodel_types.message -> string +(** [msg message] Get the minimum RBAC role required to run the procedure of the input message. + This function ignores internal roles. + If no matching role is found, the string "Not Applicable" is returned + @param message Input message. + @return string the name of the RBAC role if a matching one is found, "Not Applicable" otherwise. *) + val gen_param_groups : Datamodel_types.message -> Datamodel_types.param list From e8e589f57312a71868acdc278020d60dba1591b6 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 9 Jan 2024 15:42:53 +0000 Subject: [PATCH 018/149] CP-45888: Bump Java SDK dependencies to latest stable versions `jackson-databind` to `2.16.1` and `httpclient5` to `5.4` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/autogen/xen-api/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml index c78ec926540..abaed44d833 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml +++ b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml @@ -55,12 +55,12 @@ com.fasterxml.jackson.core jackson-databind - 2.15.1 + 2.16.1 org.apache.httpcomponents.client5 httpclient5 - 5.2.1 + 5.3 From b876a3ea429850593ea9bf54388aea3003657373 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 10 Jan 2024 08:39:18 +0000 Subject: [PATCH 019/149] Apply misc formatting and style changes - Shorten docs for Java's `IOException` - Use `String.concat` instead of multiple `^` calls in Java's `main.ml` - Rename `pascal_case_` to `pascal_case_rec` to avoid confusion for the fuinction's consumers - Use `{| |}` to replace escaped quotes when possible in Java's `main.ml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 18 +++++++----------- ocaml/sdk-gen/powershell/common_functions.ml | 6 +++--- ocaml/sdk-gen/powershell/common_functions.mli | 2 +- .../powershell/gen_powershell_binding.ml | 12 ++++++------ 4 files changed, 17 insertions(+), 21 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index b45f02c2fc8..a79ffd724f8 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -234,9 +234,8 @@ let gen_method file cls message params async_version = ) ; ("XenAPIException", "if the call failed.") ; ( "IOException" - , "if an I/O error occurs when sending or receiving, includes cases when \ - the request's payload or the response's payload cannot be written or \ - read as valid JSON." + , "if an error occurs during a send or receive. This includes cases \ + where a payload is invalid JSON." ) ] in @@ -420,7 +419,7 @@ and gen_record_tostring_contents file prefix = function let field_default = function | SecretString | String -> - "\"\"" + {|""|} | Int -> "0" | Float -> @@ -438,7 +437,7 @@ let field_default = function | Map (t1, t2) -> sprintf "new HashMap<%s, %s>()" (get_java_type t1) (get_java_type t2) | Ref ty -> - sprintf "new %s(\"OpaqueRef:NULL\")" (class_case ty) + sprintf {|new %s("OpaqueRef:NULL")|} (class_case ty) | Record _ -> assert false | Option _ -> @@ -611,15 +610,12 @@ let gen_enum file name ls = global_replace (regexp_string "\n") "\n * " escaped_description in let comment = - " /**\n" - ^ " * " - ^ final_description - ^ "\n" - ^ " */\n" + String.concat "\n" + [" /**"; " * " ^ final_description; " */"] in let json_property = if name != "UNRECOGNIZED" then - "@JsonProperty(\"" ^ name ^ "\")" + {|@JsonProperty("|} ^ name ^ {|"|} else "@JsonEnumDefaultValue" in diff --git a/ocaml/sdk-gen/powershell/common_functions.ml b/ocaml/sdk-gen/powershell/common_functions.ml index fd47311eba5..edff4bf5c70 100644 --- a/ocaml/sdk-gen/powershell/common_functions.ml +++ b/ocaml/sdk-gen/powershell/common_functions.ml @@ -8,7 +8,7 @@ open Datamodel_types open CommonFunctions module DU = Datamodel_utils -let rec pascal_case_ s = +let rec pascal_case_rec s = let ss = Astring.String.cuts ~sep:"_" ~empty:true s |> List.map String.capitalize_ascii @@ -30,7 +30,7 @@ let rec pascal_case_ s = h' ^ String.concat "" tl and pascal_case s = - let str = pascal_case_ s in + let str = pascal_case_rec s in if String.starts_with ~prefix:"set" (String.lowercase_ascii str) || String.starts_with ~prefix:"get" (String.lowercase_ascii str) @@ -197,7 +197,7 @@ and get_http_action_stem name = let parts = Astring.String.cuts ~sep:"_" name in let filtered = List.filter trim_http_action_stem parts in let trimmed = String.concat "_" filtered in - match trimmed with "" -> pascal_case_ "vm" | _ -> pascal_case_ trimmed + match trimmed with "" -> pascal_case_rec "vm" | _ -> pascal_case_rec trimmed and trim_http_action_stem x = match x with diff --git a/ocaml/sdk-gen/powershell/common_functions.mli b/ocaml/sdk-gen/powershell/common_functions.mli index f947139bec0..b13bd53aea2 100644 --- a/ocaml/sdk-gen/powershell/common_functions.mli +++ b/ocaml/sdk-gen/powershell/common_functions.mli @@ -14,7 +14,7 @@ val get_common_verb_category : string -> string @param verb - HTTP action verb. @return Common verb category. *) -val pascal_case_ : string -> string +val pascal_case_rec : string -> string (** Recursively converts a string to PascalCase. @param s - String to convert. @return PascalCase formatted string. *) diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 96d0782bf67..26b94fba23e 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -171,14 +171,14 @@ and gen_arg_param = function else "" ) - (pascal_case_ x) + (pascal_case_rec x) | Int64_query_arg x -> sprintf "\n [Parameter]\n public long? %s { get; set; }\n" - (pascal_case_ x) + (pascal_case_rec x) | Bool_query_arg x -> let y = if x = "host" then "is_host" else x in sprintf "\n [Parameter]\n public bool? %s { get; set; }\n" - (pascal_case_ y) + (pascal_case_rec y) | Varargs_query_arg -> sprintf "\n\ @@ -214,12 +214,12 @@ and gen_call_arg_params args = and gen_call_arg_param = function | String_query_arg x -> - sprintf ", %s" (pascal_case_ x) + sprintf ", %s" (pascal_case_rec x) | Int64_query_arg x -> - sprintf ", %s" (pascal_case_ x) + sprintf ", %s" (pascal_case_rec x) | Bool_query_arg x -> let y = if x = "host" then "is_host" else x in - sprintf ", %s" (pascal_case_ y) + sprintf ", %s" (pascal_case_rec y) | Varargs_query_arg -> sprintf ", Args" From 83994a9b37e45d5b378a31c9cb6a3c5bb1b449c8 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 10 Jan 2024 10:13:06 +0000 Subject: [PATCH 020/149] CP-45888: Use public setters for client settings in `JsonRpcClient` - Deprecates `Connection` constructors that take in `requestTimeout` or `connectionTimeout` - Add docs to tell user to use `Connection.client#setXYZ` instead - Add setters to `JsonRpcClient` - Replace `protected` with `private` in `JsonRpcClient` when possible, since API consumers will now be able to access `Connection.client` - Make `Connection.client` `public` - Fix misc formatting in `Connection` Signed-off-by: Danilo Del Busso --- .../java/com/xensource/xenapi/Connection.java | 93 ++++++++++++++----- .../com/xensource/xenapi/JsonRpcClient.java | 93 ++++++++++++------- 2 files changed, 134 insertions(+), 52 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index 6005982aaa6..356d5dbfe22 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -33,9 +33,11 @@ import com.xensource.xenapi.Types.BadServerResponse; import com.xensource.xenapi.Types.XenAPIException; import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.core5.util.Timeout; import java.io.IOException; import java.net.URL; +import java.util.concurrent.TimeUnit; /** * Represents a connection to a XenServer. Creating a new instance of this class initialises a new JsonRpcClient that is @@ -43,7 +45,8 @@ * method call, and dispatches it on the Connection's client via the dispatch method. */ public class Connection { - private final JsonRpcClient client; + + public final JsonRpcClient client; private APIVersion apiVersion; /** * The opaque reference to the session used by this connection @@ -75,12 +78,17 @@ public Connection(JsonRpcClient jsonRpcClient) { * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. * - * @param httpClient The HttpClient used to make calls, this will be used by JsonRpcClient for handling requests - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param httpClient The HttpClient used to make calls, this will be used by the underlying {@link #client} for handling requests + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. * @param requestTimeout The reply timeout for JSON-RPC calls in seconds + * @deprecated This constructor is deprecated. To set the {@code requestTimeout} please {@link #setRequestTimeout(int)}. You may also use the {@link com.xensource.xenapi.JsonRpcClient#setRequestTimeout(int)} + * method of this object's {@link #client}. This option is only advisable if you are managing your own {@link com.xensource.xenapi.JsonRpcClient} as the underlying + * {@link #client} for this object. */ + @Deprecated public Connection(CloseableHttpClient httpClient, URL url, int requestTimeout) { - this.client = new JsonRpcClient(httpClient, url, requestTimeout); + this.client = new JsonRpcClient(httpClient, url); + this.client.setRequestTimeout(requestTimeout); } /** @@ -96,7 +104,7 @@ public Connection(CloseableHttpClient httpClient, URL url, int requestTimeout) { * This constructor uses the default values of the reply and connection timeouts for the JSON-RPC calls * (600 seconds and 5 seconds respectively). * - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. */ public Connection(URL url) { this.client = new JsonRpcClient(url); @@ -112,12 +120,19 @@ public Connection(URL url) { * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. * - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. * @param requestTimeout The reply timeout for JSON-RPC calls in seconds * @param connectionTimeout The connection timeout for JSON-RPC calls in seconds + * @deprecated This constructor is deprecated. To set {@code requestTimeout} or {@code connectionTimeout} please use {@link #setRequestTimeout(int)} or {@link #setConnectionTimeout(int)} respectively. + * You may also use the {@link com.xensource.xenapi.JsonRpcClient#setRequestTimeout(int)} method of this object's {@link #client}. + * This option is only advisable if you are managing your own {@link com.xensource.xenapi.JsonRpcClient} as the underlying + * {@link #client} for this object. */ + @Deprecated public Connection(URL url, int requestTimeout, int connectionTimeout) { - this.client = new JsonRpcClient(url, requestTimeout, connectionTimeout); + this.client = new JsonRpcClient(url); + this.client.setRequestTimeout(requestTimeout); + this.client.setConnectionTimeout(connectionTimeout); } /** @@ -133,7 +148,7 @@ public Connection(URL url, int requestTimeout, int connectionTimeout) { * This constructor uses the default values of the reply and connection timeouts for the JSON-RPC calls * (600 seconds and 5 seconds respectively). * - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. * @param sessionReference A reference to a logged-in Session. Any method calls on this * Connection will use it. This constructor does not call Session.loginWithPassword, and dispose() on the resulting * Connection object does not call Session.logout. The programmer is responsible for ensuring the Session is logged @@ -154,19 +169,52 @@ public Connection(URL url, String sessionReference) { * When this constructor is used, a call to dispose() will do nothing. The programmer is responsible for manually * logging out the Session. * - * @param url The URL of the server to connect to. Should be of the form http(s)://host-url./jsonrpc or http(s)://host-url. + * @param url The URL of the server to connect to. Should be of the form http(s)://host-url/jsonrpc or http(s)://host-url. * @param sessionReference A reference to a logged-in Session. Any method calls on this Connection will use it. * This constructor does not call Session.loginWithPassword, and dispose() on the resulting * Connection object does not call Session.logout. The programmer is responsible for * ensuring the Session is logged in and out correctly. * @param requestTimeout The reply timeout for JSON-RPC calls in seconds * @param connectionTimeout The connection timeout for JSON-RPC calls in seconds + * @deprecated This constructor is deprecated. To set {@code requestTimeout} or {@code connectionTimeout} please use {@link #setRequestTimeout(int)} or {@link #setConnectionTimeout(int)} respectively. + * You may also use the {@link com.xensource.xenapi.JsonRpcClient#setRequestTimeout(int)} method of this object's {@link #client}. + * This option is only advisable if you are managing your own {@link com.xensource.xenapi.JsonRpcClient} as the underlying + * {@link #client} for this object. */ + @Deprecated public Connection(URL url, String sessionReference, int requestTimeout, int connectionTimeout) { - this.client = new JsonRpcClient(url, requestTimeout, connectionTimeout); + this.client = new JsonRpcClient(url); + this.client.setRequestTimeout(requestTimeout); + this.client.setConnectionTimeout(connectionTimeout); this.sessionReference = sessionReference; } + /** + * Set the timeout in seconds for every request made by this object's {@link #client}. + * If not set the value defaults to {@value JsonRpcClient#DEFAULT_REQUEST_TIMEOUT}. + * You may also pass your own {@link JsonRpcClient} in the constructor for more control. + * + * @param requestTimeout the timeout value in seconds + * @throws NullPointerException if the {@link #client} is null + * @see org.apache.hc.client5.http.config.RequestConfig.Builder#setConnectionRequestTimeout(long, TimeUnit) + */ + public void setRequestTimeout(int requestTimeout) throws NullPointerException { + this.client.setRequestTimeout(requestTimeout); + } + + /** + * Set the connection timeout in seconds for its {@link #client}'s {@link org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager}. + * If not set the value defaults to {@value JsonRpcClient#DEFAULT_CONNECTION_TIMEOUT}. + * You may also pass your own {@link JsonRpcClient} in the constructor for more control. + * + * @param connectionTimeout the client's connection timeout in seconds. + * @throws NullPointerException if the {@link #client} is null + * @see org.apache.hc.client5.http.config.ConnectionConfig.Builder#setConnectTimeout(Timeout) + */ + public void setConnectionTimeout(int connectionTimeout) { + this.client.setConnectionTimeout(connectionTimeout); + } + /** * Updated when Session.login_with_password() is called. */ @@ -176,9 +224,12 @@ public APIVersion getAPIVersion() { private void setAPIVersion(Session session) throws IOException { try { - long major = session.getThisHost(this).getAPIVersionMajor(this); - long minor = session.getThisHost(this).getAPIVersionMajor(this); - apiVersion = APIVersion.fromMajorMinor(major, minor); + var pools = Pool.getAllRecords(this); + var pool = pools.values().stream().findFirst(); + if (pool.isPresent()) { + var host = pool.get().master.getRecord(this); + apiVersion = APIVersion.fromMajorMinor(host.APIVersionMajor, host.APIVersionMinor); + } } catch (BadServerResponse exn) { apiVersion = APIVersion.UNKNOWN; } @@ -196,13 +247,13 @@ public String getSessionReference() { /** * Send a method call to xapi's backend. You need to provide the type of the data returned by a successful response. * - * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password - * @param methodParameters the methodParameters of the method call - * @param responseTypeReference the type of the response, wrapped with a TypeReference + * @param methodCall The JSON-RPC xapi method call. e.g.: session.login_with_password + * @param methodParameters The methodParameters of the method call + * @param responseTypeReference The type of the response, wrapped with a TypeReference * @param The type of the response's payload. For instance, a map of opaque references to VM objects is expected when calling VM.get_all_records - * @return The result of the call with the type specified under T. - * @throws XenAPIException if the call failed. - * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. + * @return The result of the call with the type specified under T. + * @throws XenAPIException if the call failed. + * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. */ public T dispatch(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws XenAPIException, IOException { var result = client.sendRequest(methodCall, methodParameters, responseTypeReference); @@ -228,8 +279,8 @@ public T dispatch(String methodCall, Object[] methodParameters, TypeReferenc * * @param methodCall the JSON-RPC xapi method call. e.g.: session.login_with_password * @param methodParameters the methodParameters of the method call - * @throws XenAPIException if the call failed. - * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. + * @throws XenAPIException if the call failed. + * @throws IOException if an I/O error occurs when sending or receiving, includes cases when the request's payload or the response's payload cannot be written or read as valid JSON. */ public void dispatch(String methodCall, Object[] methodParameters) throws XenAPIException, IOException { var typeReference = new TypeReference() { diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index 7172e148f4c..346ea62aba1 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -41,6 +41,7 @@ import org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager; import org.apache.hc.core5.http.ContentType; import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.util.Timeout; import java.io.IOException; import java.net.URL; @@ -57,8 +58,10 @@ * The client can be customised by passing it as a parameter to corresponding constructor, enabling custom * handling of requests. *
- * By default, the timeout for requests is set to 10 minutes (600 seconds). The default timeout for connecting to the - * JSON-RPC backend is set to 5 seconds. + *
+ * By default, the timeout for requests is set to {@value #DEFAULT_REQUEST_TIMEOUT}. The default timeout for connecting to the + * JSON-RPC backend is set to {@value #DEFAULT_CONNECTION_TIMEOUT} seconds. The maximum number of concurrent connections handled + * by the underlying {@link PoolingHttpClientConnectionManager} is {@value #MAX_CONCURRENT_CONNECTIONS}. * * @see CloseableHttpClient CloseableHttpClient is used to make requests and connect to the backend * @see ObjectMapper ObjectMapper is used to marshall requests and responses @@ -67,18 +70,20 @@ public class JsonRpcClient { private static final int DEFAULT_REQUEST_TIMEOUT = 600; private static final int DEFAULT_CONNECTION_TIMEOUT = 5; - protected static final int MAX_CONCURRENT_CONNECTIONS = 10; - - protected static final String JSON_BACKEND_PATH = "/jsonrpc"; + private static final int MAX_CONCURRENT_CONNECTIONS = 10; - protected final CloseableHttpClient httpClient; - protected final String jsonRpcBackendUrl; - protected final ObjectMapper objectMapper; - protected final int requestTimeout; + private static final String JSON_BACKEND_PATH = "/jsonrpc"; + private final CloseableHttpClient httpClient; + private final String jsonRpcBackendUrl; + private final ObjectMapper objectMapper; private final RequestConfig defaultRequestConfig = RequestConfig.custom() .setCookieSpec(StandardCookieSpec.IGNORE) .build(); + private int requestTimeout; + private PoolingHttpClientConnectionManager connectionManager; + + //region Constructors /** * Create a JsonRpcClient with default settings. @@ -87,24 +92,12 @@ public class JsonRpcClient { * @see JsonRpcClient JsonRpcClient for more info on using this class */ public JsonRpcClient(URL jsonRpcBackendUrl) { - this(jsonRpcBackendUrl, DEFAULT_REQUEST_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT); - } - - /** - * Create a JsonRpcClient with the option to define the request and connection timeout values. - * - * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. - * @param requestTimeout the timeout value for requests. - * @param connectionTimeout the timeout value for the initial connection to the host. - * @see JsonRpcClient JsonRpcClient for more info on using this class - */ - public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTimeout) { var connectionConfig = ConnectionConfig .custom() - .setConnectTimeout(connectionTimeout, TimeUnit.SECONDS) + .setConnectTimeout(DEFAULT_CONNECTION_TIMEOUT, TimeUnit.SECONDS) .build(); - var connectionManager = new PoolingHttpClientConnectionManager(); + this.connectionManager = new PoolingHttpClientConnectionManager(); connectionManager.setDefaultConnectionConfig(connectionConfig); connectionManager.setMaxTotal(MAX_CONCURRENT_CONNECTIONS); @@ -113,7 +106,7 @@ public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTi .setConnectionManager(connectionManager) .build(); this.jsonRpcBackendUrl = formatBackendUrl(jsonRpcBackendUrl); - this.requestTimeout = requestTimeout; + this.requestTimeout = DEFAULT_REQUEST_TIMEOUT; this.objectMapper = new ObjectMapper(); initializeObjectMapperConfiguration(); } @@ -123,20 +116,58 @@ public JsonRpcClient(URL jsonRpcBackendUrl, int requestTimeout, int connectionTi * * @param client the custom HttpClient to use for all requests * @param jsonRpcBackendUrl the URL of the JSON-RPC backend. Usually of the form https://<address>. - * @param requestTimeout the timeout value for requests. * @see CloseableHttpClient CloseableHttpClient the client that will be used for dispatching requests * @see JsonRpcClient JsonRpcClient for more info on using this class */ - public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requestTimeout) { + public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl) { httpClient = client; - - this.requestTimeout = requestTimeout; this.jsonRpcBackendUrl = formatBackendUrl(jsonRpcBackendUrl); - this.objectMapper = new ObjectMapper(); initializeObjectMapperConfiguration(); } + //endregion + + //region Public Setters + + /** + * Set the timeout in seconds for every request made by this client. + * If not set the value defaults to {@value #DEFAULT_REQUEST_TIMEOUT}. + * + * @param requestTimeout the timeout value in seconds + * @see org.apache.hc.client5.http.config.RequestConfig.Builder#setConnectionRequestTimeout(long, TimeUnit) + */ + public void setRequestTimeout(int requestTimeout) { + this.requestTimeout = requestTimeout; + } + + /** + * Set the connection timeout in seconds for this client's {@link org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager}. + * If not set the value defaults to {@value #DEFAULT_CONNECTION_TIMEOUT}. + * + * @param connectionTimeout the client's connection timeout in seconds. + * @see org.apache.hc.client5.http.config.ConnectionConfig.Builder#setConnectTimeout(Timeout) + */ + public void setConnectionTimeout(int connectionTimeout) { + connectionManager.setDefaultConnectionConfig(ConnectionConfig + .custom() + .setConnectTimeout(connectionTimeout, TimeUnit.SECONDS) + .build() + ); + } + + /** + * Set the maximum number of connections that this client's {@link org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager} will keep open. + * If not set the value defaults to {@value #MAX_CONCURRENT_CONNECTIONS}. + * + * @param maxConcurrentConnections the maximum number of connections managed by the connection manager + * @see org.apache.hc.core5.pool.ConnPoolControl#setMaxTotal(int) + */ + public void setMaxConcurrentConnections(int maxConcurrentConnections) { + connectionManager.setMaxTotal(maxConcurrentConnections); + } + //endregion + /** * Send a method call to xapi's backend. You need to provide the type of the data returned by a successful response. * @@ -144,11 +175,11 @@ public JsonRpcClient(CloseableHttpClient client, URL jsonRpcBackendUrl, int requ * @param methodParameters the parameters of the method call * @param responseTypeReference the type of the response, wrapped with a TypeReference * @param The type of the response's payload. For instance, a map of opaque references to VM objects is expected when calling VM.get_all_records - * @return a JsonRpcResponse object. If its error field is empty, the response was successful. + * @return a {@link JsonRpcResponse} object. If its error field is empty, the response was successful. * @throws JsonProcessingException if the request's payload or the response's payload cannot be written or read as valid JSON * @throws IOException if an I/O error occurs when sending or receiving */ - public JsonRpcResponse sendRequest(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws IOException { + protected JsonRpcResponse sendRequest(String methodCall, Object[] methodParameters, TypeReference responseTypeReference) throws IOException { var requestBody = objectMapper .writeValueAsString(new JsonRpcRequest(methodCall, methodParameters)); From 1cf6b2d50074ed513f50c94190f9b116314fad9b Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 10 Jan 2024 10:28:21 +0000 Subject: [PATCH 021/149] CP-45888: Use `Pool.getAllRecords` and `Host.getRecord` to set API version This version replicates what we do in the C# SDK, and uses fewer calls to fetch the API version Signed-off-by: Danilo Del Busso --- .../src/main/java/com/xensource/xenapi/Connection.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java index 356d5dbfe22..53bdc1d8b83 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/Connection.java @@ -222,7 +222,8 @@ public APIVersion getAPIVersion() { return apiVersion; } - private void setAPIVersion(Session session) throws IOException { + private void setAPIVersion() throws IOException { + apiVersion = APIVersion.UNKNOWN; try { var pools = Pool.getAllRecords(this); var pool = pools.values().stream().findFirst(); @@ -231,7 +232,7 @@ private void setAPIVersion(Session session) throws IOException { apiVersion = APIVersion.fromMajorMinor(host.APIVersionMajor, host.APIVersionMinor); } } catch (BadServerResponse exn) { - apiVersion = APIVersion.UNKNOWN; + // ignore, we default to UNKNOWN } } @@ -264,7 +265,7 @@ public T dispatch(String methodCall, Object[] methodParameters, TypeReferenc if (methodCall.equals("session.login_with_password")) { var session = ((Session) result.result); sessionReference = session.ref; - setAPIVersion(session); + setAPIVersion(); } else if (methodCall.equals("session.slave_local_login_with_password")) { var session = ((Session) result.result); sessionReference = session.ref; From 517ef1a5c3bb4b6822e182a24eed6470d22e4142 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 15 Jan 2024 15:47:18 +0000 Subject: [PATCH 022/149] CP-45888: Fix incorrect URL presence check in `JsonRpcClient` Signed-off-by: Danilo Del Busso --- .../src/main/java/com/xensource/xenapi/JsonRpcClient.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index 346ea62aba1..e3513908116 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -222,7 +222,7 @@ private String formatBackendUrl(URL url) { // We only replace it when it's empty. // If the user purposely set the path // we use the given value even if incorrect - if (!url.getPath().isEmpty()) { + if (url.getPath().isEmpty()) { return url.getProtocol() + "://" + url.getHost() + JSON_BACKEND_PATH; } return url.toString(); From 9677795da60f6723129819df4e66ec1c82f396d3 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Mon, 15 Jan 2024 15:48:44 +0000 Subject: [PATCH 023/149] CP-45888: Restore `toX` methods in `Types.java` - Restore use of `Set` instead of `HashSet` to avoid consumer implementation breaking - Remove use of raw types to avoid runtime exceptions - Deprecate `toX(Object)` methods in `Types` since they should be internal - Restore `toX(Object, Connection)` methods - Supress unchecked cast warnings with annotation to remove them during compilation. Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/main.ml | 251 +++++++++++++++++++++++++++++++++++-- 1 file changed, 243 insertions(+), 8 deletions(-) diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index a79ffd724f8..254ce1c35f8 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -142,7 +142,7 @@ let rec get_java_type ty = Hashtbl.replace enums name ls ; sprintf "Types.%s" (class_case name) | Set t1 -> - sprintf "HashSet<%s>" (get_java_type t1) + sprintf "Set<%s>" (get_java_type t1) | Map (t1, t2) -> sprintf "Map<%s, %s>" (get_java_type t1) (get_java_type t2) | Ref x -> @@ -157,6 +157,36 @@ let rec get_java_type ty = let switch_enum = Enum ("XenAPIObjects", List.map (fun x -> (x.name, x.description)) classes) +(*Helper function for get_marshall_function*) +let rec get_marshall_function_rec = function + | SecretString | String -> + "String" + | Int -> + "Long" + | Float -> + "Double" + | Bool -> + "Boolean" + | DateTime -> + "Date" + | Enum (name, _) -> + class_case name + | Set t1 -> + sprintf "SetOf%s" (get_marshall_function_rec t1) + | Map (t1, t2) -> + sprintf "MapOf%s%s" + (get_marshall_function_rec t1) + (get_marshall_function_rec t2) + | Ref ty -> + class_case ty (* We want to hide all refs *) + | Record ty -> + sprintf "%sRecord" (class_case ty) + | Option ty -> + get_marshall_function_rec ty + +(*get_marshall_function (Set(Map(Float,Bool)));; -> "toSetOfMapOfDoubleBoolean"*) +let get_marshall_function ty = "to" ^ get_marshall_function_rec ty + let _ = get_java_type switch_enum (* Generate the methods *) @@ -321,8 +351,7 @@ let gen_method file cls message params async_version = List.iter (fun {param_name= s; _} -> let name = camel_case s in - fprintf file " Map %s_map = %s.toMap();\n" name - name + fprintf file " var %s_map = %s.toMap();\n" name name ) record_params ; @@ -484,7 +513,7 @@ let gen_record file cls = fprintf file " * Convert a %s.Record to a Map\n" cls.name ; fprintf file " */\n" ; fprintf file " public Map toMap() {\n" ; - fprintf file " Map map = new HashMap<>();\n" ; + fprintf file " var map = new HashMap();\n" ; List.iter (gen_record_tomap_contents file []) contents ; if cls.name = "event" then @@ -596,6 +625,194 @@ import java.io.IOException; fprintf file "}" ; close_out file +(**?*) +(* Generate Marshalling Class *) + +(*This generates the special case code for marshalling the snapshot field in an Event.Record*) +let generate_snapshot_hack file = + fprintf file "\n" ; + fprintf file "\n" ; + fprintf file " Object a,b;\n" ; + fprintf file " a=map.get(\"snapshot\");\n" ; + fprintf file " switch(%s(record.clazz))\n" + (get_marshall_function switch_enum) ; + fprintf file " {\n" ; + List.iter + (fun x -> + fprintf file " case %17s: b = %25s(a); break;\n" + (String.uppercase_ascii x) + (get_marshall_function (Record x)) + ) + (List.map + (fun x -> x.name) + (List.filter (fun x -> not (class_is_empty x)) classes) + ) ; + fprintf file + " default: throw new RuntimeException(\"Internal error in \ + auto-generated code whilst unmarshalling event snapshot\");\n" ; + fprintf file " }\n" ; + fprintf file " record.snapshot = b;\n" + +let gen_marshall_record_field file prefix field = + let ty = get_marshall_function field.ty in + let name = String.concat "_" (List.rev (field.field_name :: prefix)) in + let name' = camel_case name in + fprintf file " record.%s = %s(map.get(\"%s\"));\n" name' ty name + +let rec gen_marshall_record_namespace file prefix (name, contents) = + List.iter (gen_marshall_record_contents file (name :: prefix)) contents + +and gen_marshall_record_contents file prefix = function + | Field f -> + gen_marshall_record_field file prefix f + | Namespace (n, cs) -> + gen_marshall_record_namespace file prefix (n, cs) ; + () + +(*Every type which may be returned by a function may also be the result of the*) +(* corresponding asynchronous task. We therefore need to generate corresponding*) +(* marshalling functions which can take the raw xml of the tasks result field*) +(* and turn it into the corresponding type. Luckily, the only things returned by*) +(* asynchronous tasks are object references and strings, so rather than implementing*) +(* the general recursive structure we'll just make one for each of the classes*) +(* that's been registered as a marshall-needing type*) + +let generate_reference_task_result_func file clstr = + fprintf file {| /** + * Attempt to convert the {@link Task}'s result to a {@link %s} object. + * Will return null if the method cannot fetch a valid value from the {@link Task} object. + * @param task The task from which to fetch the result. + * @param connection The connection + * @return the instantiated object if a valid value was found, null otherwise. + * @throws BadServerResponse Thrown if the response from the server contains an invalid status. + * @throws XenAPIException if the call failed. + * @throws IOException if an error occurs during a send or receive. This includes cases where a payload is invalid JSON. + */ +|} clstr; + fprintf file + " public static %s to%s(Task task, Connection connection) throws IOException {\n" + clstr clstr ; + fprintf file + " return Types.to%s(task.getResult(connection));\n" + clstr ; + fprintf file " }\n" ; + fprintf file "\n" + +let gen_task_result_func file = function + | Ref ty -> + generate_reference_task_result_func file (class_case ty) + | _ -> + () + +(*don't generate for complicated types. They're not needed.*) + +let rec gen_marshall_body file = function + | SecretString | String -> + fprintf file " return (String) object;\n" + | Int -> + fprintf file " return Long.valueOf((String) object);\n" + | Float -> + fprintf file " return (Double) object;\n" + | Bool -> + fprintf file " return (Boolean) object;\n" + | DateTime -> + fprintf file + " try {\n\ + \ return (Date) object;\n\ + \ } catch (ClassCastException e){\n\ + \ //Occasionally the date comes back as an ocaml float \ + rather than\n\ + \ //in the xmlrpc format! Catch this and convert.\n\ + \ return (new Date((long) (1000*Double.parseDouble((String) \ + object))));\n\ + \ }\n" + | Ref ty -> + fprintf file " return new %s((String) object);\n" (class_case ty) + | Enum (name, _) -> + fprintf file " try {\n" ; + fprintf file + " return %s.valueOf(((String) \ + object).toUpperCase().replace('-','_'));\n" + (class_case name) ; + fprintf file " } catch (IllegalArgumentException ex) {\n" ; + fprintf file " return %s.UNRECOGNIZED;\n" (class_case name) ; + fprintf file " }\n" + | Set ty -> + let ty_name = get_java_type ty in + let marshall_fn = get_marshall_function ty in + fprintf file " Object[] items = (Object[]) object;\n" ; + fprintf file " Set<%s> result = new LinkedHashSet<>();\n" ty_name ; + fprintf file " for(Object item: items) {\n" ; + fprintf file " %s typed = %s(item);\n" ty_name marshall_fn ; + fprintf file " result.add(typed);\n" ; + fprintf file " }\n" ; + fprintf file " return result;\n" + | Map (ty, ty') -> + let ty_name = get_java_type ty in + let ty_name' = get_java_type ty' in + let marshall_fn = get_marshall_function ty in + let marshall_fn' = get_marshall_function ty' in + fprintf file " var map = (Map)object;\n" ; + fprintf file " var result = new HashMap<%s,%s>();\n" ty_name + ty_name' ; + fprintf file " for(var entry: map.entrySet()) {\n" ; + fprintf file " var key = %s(entry.getKey());\n" marshall_fn ; + fprintf file " var value = %s(entry.getValue());\n" + marshall_fn' ; + fprintf file " result.put(key, value);\n" ; + fprintf file " }\n" ; + fprintf file " return result;\n" + | Record ty -> + let contents = Hashtbl.find records ty in + let cls_name = class_case ty in + fprintf file + " Map map = (Map) object;\n" ; + fprintf file " %s.Record record = new %s.Record();\n" cls_name + cls_name ; + List.iter (gen_marshall_record_contents file []) contents ; + (*Event.Record needs a special case to handle snapshots*) + if ty = "event" then generate_snapshot_hack file ; + fprintf file " return record;\n" + | Option ty -> + gen_marshall_body file ty + +let rec gen_marshall_func file ty = + match ty with + | Option x -> + if TypeSet.mem x !types then + () + else + gen_marshall_func file ty + | _ -> + let type_string = get_java_type ty in + fprintf file {| /** + * Converts an {@link Object} to a {@link %s} object. + *
+ * This method takes an {@link Object} as input and attempts to convert it into a {@link %s} object. + * If the input object is null, the method returns null. Otherwise, it creates a new {@link %s} + * object using the input object's {@link String} representation. + *
+ * @param object The {@link Object} to be converted to a {@link %s} object. + * @return A {@link %s} object created from the input {@link Object}'s {@link String} representation, + * or null if the input object is null. + * @deprecated this method will not be publicly exposed in future releases of this package. + */ + @Deprecated +|} type_string type_string type_string type_string type_string ; + let fn_name = get_marshall_function ty in + + if match ty with | Map _ | Record _ -> true | _ -> false then + fprintf file " @SuppressWarnings(\"unchecked\")\n"; + + fprintf file " public static %s %s(Object object) {\n" type_string + fn_name ; + fprintf file " if (object == null) {\n" ; + fprintf file " return null;\n" ; + fprintf file " }\n" ; + gen_marshall_body file ty ; + fprintf file " }\n\n" +(***) + let gen_enum file name ls = let name = class_case name in let ls = @@ -615,11 +832,11 @@ let gen_enum file name ls = in let json_property = if name != "UNRECOGNIZED" then - {|@JsonProperty("|} ^ name ^ {|"|} + {|@JsonProperty("|} ^ name ^ {|")|} else "@JsonEnumDefaultValue" in - comment ^ " " ^ json_property ^ "\n" ^ " " ^ enum_of_wire name + comment ^ " \n" ^ json_property ^ "\n" ^ " " ^ enum_of_wire name in fprintf file "%s" (String.concat ",\n" (List.map to_member_declaration ls)) ; fprintf file ";\n" ; @@ -700,7 +917,7 @@ let gen_types_class folder = print_license file ; fprintf file {|package com.xensource.xenapi; -import java.util.Map; +import java.util.*; import com.fasterxml.jackson.annotation.JsonEnumDefaultValue; import com.fasterxml.jackson.annotation.JsonProperty; import java.io.IOException; @@ -760,6 +977,7 @@ public class Types return sb.toString(); } } + /** * Thrown if the response from the server contains an invalid status. */ @@ -802,7 +1020,24 @@ public class Types fprintf file "\n" ; Hashtbl.iter (gen_error file) Datamodel.errors ; fprintf file "\n" ; - fprintf file "}\n" + TypeSet.iter (gen_marshall_func file) !types ; + fprintf file "\n" ; + TypeSet.iter (gen_task_result_func file) !types ; + fprintf file +{| + public static EventBatch toEventBatch(Object object) { + if (object == null) { + return null; + } + Map map = (Map) object; + EventBatch batch = new EventBatch(); + batch.token = toString(map.get("token")); + batch.validRefCounts = map.get("valid_ref_counts"); + batch.events = toSetOfEventRecord(map.get("events")); + return batch; + } +} +|} (* Now run it *) From 102c825bc5a50c14a171596391c1041ea58ec424 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 16 Jan 2024 10:37:43 +0000 Subject: [PATCH 024/149] CP-45888: Improve deprecation info Adds "@Deprecated (since = x )" annotations, and populates it for methods and fields. Also adds "@deprecated xyz" in method JavaDocs. Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/common/CommonFunctions.ml | 3 + ocaml/sdk-gen/common/CommonFunctions.mli | 6 ++ .../java/com/xensource/xenapi/EventBatch.java | 2 + ocaml/sdk-gen/java/main.ml | 79 +++++++++++++------ 4 files changed, 64 insertions(+), 26 deletions(-) diff --git a/ocaml/sdk-gen/common/CommonFunctions.ml b/ocaml/sdk-gen/common/CommonFunctions.ml index fe89dd9600e..12ef3420d31 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.ml +++ b/ocaml/sdk-gen/common/CommonFunctions.ml @@ -122,6 +122,9 @@ let get_prototyped_release lifecycle = let get_published_release lifecycle = lifecycle_matcher Lifecycle.Published lifecycle +let get_deprecated_release lifecycle = + lifecycle_matcher Lifecycle.Published lifecycle + let get_release_branding codename = try let found = diff --git a/ocaml/sdk-gen/common/CommonFunctions.mli b/ocaml/sdk-gen/common/CommonFunctions.mli index 2b2312fafb7..9a88b5cd5bd 100644 --- a/ocaml/sdk-gen/common/CommonFunctions.mli +++ b/ocaml/sdk-gen/common/CommonFunctions.mli @@ -4,6 +4,12 @@ exception Unknown_wire_protocol (** Type representing supported protocols. *) type wireProtocol = XmlRpc | JsonRpc +val get_deprecated_release : + (Datamodel_types.Lifecycle.change * string * 'a) list -> string +(** [get_deprecated_release codename] Gets the non-branded release name for a lifecycle if it's deprecated + @param lifecycle The lifecycle transitions to check. + @return The non-branded release name for a lifecycle if it's deprecated, empty if not deprecated *) + val get_release_branding : string -> string (** [get_release_branding codename] Gets the branding for a release codename. @param codename Release codename to lookup. diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/EventBatch.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/EventBatch.java index 2734ca18840..e70823f2638 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/EventBatch.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/EventBatch.java @@ -30,6 +30,7 @@ package com.xensource.xenapi; import java.util.Set; +import com.fasterxml.jackson.annotation.JsonProperty; /** * Class used to map the output of Event.from(). @@ -46,6 +47,7 @@ public class EventBatch /** * The number of valid objects of all types in the database. */ + @JsonProperty("valid_ref_counts") public Object validRefCounts; /** diff --git a/ocaml/sdk-gen/java/main.ml b/ocaml/sdk-gen/java/main.ml index 254ce1c35f8..496de17a075 100644 --- a/ocaml/sdk-gen/java/main.ml +++ b/ocaml/sdk-gen/java/main.ml @@ -204,14 +204,20 @@ let get_java_type_or_void = function (* it has a self parameter or not.*) (*Similar functions for deprecation of methods*) -let get_method_deprecated message = - message.msg_release.internal_deprecated_since <> None -let get_method_deprecated_string message = - if get_method_deprecated message then - "@Deprecated" - else - "" +let get_method_deprecated_release_name message = + match message.msg_release.internal_deprecated_since with + | Some version -> + Some (get_release_branding version) + | None -> + None + +let get_method_deprecated_annotation message = + match get_method_deprecated_release_name message with + | Some version -> + {|@Deprecated(since = "|} ^ version ^ {|")|} + | None -> + "" let get_method_param {param_type= ty; param_name= name; _} = let ty = get_java_type ty in @@ -244,7 +250,6 @@ let rec range = function 0 -> [] | i -> range (i - 1) @ [i] (* Here is the main method generating function.*) let gen_method file cls message params async_version = - let deprecated_string = get_method_deprecated_string message in let return_type = if String.lowercase_ascii cls.name = "event" @@ -276,7 +281,14 @@ let gen_method file cls message params async_version = fprintf file " * Minimum allowed role: %s\n" (get_minimum_allowed_role message) ; if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; - if get_method_deprecated message then fprintf file " * @deprecated\n" ; + let deprecated_info = + match get_method_deprecated_release_name message with + | Some version -> + " * @deprecated since " ^ version ^ "\n" + | None -> + "" + in + fprintf file "%s" deprecated_info ; fprintf file " *\n" ; fprintf file " * @param c The connection the call is made on\n" ; @@ -313,13 +325,20 @@ let gen_method file cls message params async_version = ) message.msg_errors ; - fprintf file " */\n" ; + fprintf file " */\n" ; + let deprecated_string = + match get_method_deprecated_annotation message with + | "" -> + "" + | other -> + " " ^ other ^ "\n" + in if async_version then - fprintf file " %s public %sTask %sAsync(%s) throws\n" deprecated_string + fprintf file "%s public %sTask %sAsync(%s) throws\n" deprecated_string method_static method_name paramString else - fprintf file " %s public %s%s %s(%s) throws\n" deprecated_string + fprintf file "%s public %s%s %s(%s) throws\n" deprecated_string method_static return_type method_name paramString ; let all_errors = @@ -415,7 +434,12 @@ let gen_record_field file prefix field cls = if not (publishInfo = "") then fprintf file " * %s\n" publishInfo ; fprintf file " */\n" ; fprintf file " @JsonProperty(\"%s\")\n" full_name ; - fprintf file " public %s %s;\n" ty name + + if field.lifecycle.state = Lifecycle.Deprecated_s then + fprintf file " @Deprecated(since = \"%s\")\n" + (get_release_branding (get_deprecated_release field.lifecycle.transitions)) ; + + fprintf file " public %s %s;\n\n" ty name let rec gen_record_namespace file prefix (name, contents) cls = List.iter (gen_record_contents file (name :: prefix) cls) contents @@ -678,7 +702,8 @@ and gen_marshall_record_contents file prefix = function (* that's been registered as a marshall-needing type*) let generate_reference_task_result_func file clstr = - fprintf file {| /** + fprintf file + {| /** * Attempt to convert the {@link Task}'s result to a {@link %s} object. * Will return null if the method cannot fetch a valid value from the {@link Task} object. * @param task The task from which to fetch the result. @@ -688,13 +713,13 @@ let generate_reference_task_result_func file clstr = * @throws XenAPIException if the call failed. * @throws IOException if an error occurs during a send or receive. This includes cases where a payload is invalid JSON. */ -|} clstr; +|} + clstr ; fprintf file - " public static %s to%s(Task task, Connection connection) throws IOException {\n" + " public static %s to%s(Task task, Connection connection) throws \ + IOException {\n" clstr clstr ; - fprintf file - " return Types.to%s(task.getResult(connection));\n" - clstr ; + fprintf file " return Types.to%s(task.getResult(connection));\n" clstr ; fprintf file " }\n" ; fprintf file "\n" @@ -785,7 +810,8 @@ let rec gen_marshall_func file ty = gen_marshall_func file ty | _ -> let type_string = get_java_type ty in - fprintf file {| /** + fprintf file + {| /** * Converts an {@link Object} to a {@link %s} object. *
* This method takes an {@link Object} as input and attempts to convert it into a {@link %s} object. @@ -798,12 +824,13 @@ let rec gen_marshall_func file ty = * @deprecated this method will not be publicly exposed in future releases of this package. */ @Deprecated -|} type_string type_string type_string type_string type_string ; +|} + type_string type_string type_string type_string type_string ; let fn_name = get_marshall_function ty in - if match ty with | Map _ | Record _ -> true | _ -> false then - fprintf file " @SuppressWarnings(\"unchecked\")\n"; - + if match ty with Map _ | Record _ -> true | _ -> false then + fprintf file " @SuppressWarnings(\"unchecked\")\n" ; + fprintf file " public static %s %s(Object object) {\n" type_string fn_name ; fprintf file " if (object == null) {\n" ; @@ -836,7 +863,7 @@ let gen_enum file name ls = else "@JsonEnumDefaultValue" in - comment ^ " \n" ^ json_property ^ "\n" ^ " " ^ enum_of_wire name + comment ^ "\n " ^ json_property ^ "\n " ^ enum_of_wire name in fprintf file "%s" (String.concat ",\n" (List.map to_member_declaration ls)) ; fprintf file ";\n" ; @@ -1024,7 +1051,7 @@ public class Types fprintf file "\n" ; TypeSet.iter (gen_task_result_func file) !types ; fprintf file -{| + {| public static EventBatch toEventBatch(Object object) { if (object == null) { return null; From fc3ac0e8deba1ac0a33346985dd88658dc289aec Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Tue, 16 Jan 2024 11:29:37 +0000 Subject: [PATCH 025/149] CP-45888: Add a custom date deserializer to handle outliers xapi returns strings of the form "0.0" for `event.timestamp` fields. This commit adds a new custom JSON deserializer to handle such values. Signed-off-by: Danilo Del Busso --- .../xenapi/CustomDateDeserializer.java | 94 +++++++++++++++++++ .../com/xensource/xenapi/JsonRpcClient.java | 7 +- 2 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/CustomDateDeserializer.java diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/CustomDateDeserializer.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/CustomDateDeserializer.java new file mode 100644 index 00000000000..a0e9bff1a3d --- /dev/null +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/CustomDateDeserializer.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +package com.xensource.xenapi; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; + +import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; + +/** + * {@link CustomDateDeserializer} is a Jackson JSON deserializer for parsing {@link Date} objects + * from custom date formats used in Xen-API responses. + */ +public class CustomDateDeserializer extends StdDeserializer { + + /** + * Array of {@link SimpleDateFormat} objects representing the custom date formats + * used in XenServer API responses. + */ + private final SimpleDateFormat[] dateFormatters + = new SimpleDateFormat[]{ + new SimpleDateFormat("yyyyMMdd'T'HH:mm:ss'Z'"), + new SimpleDateFormat("ss.SSS") + }; + + /** + * Constructs a {@link CustomDateDeserializer} instance. + */ + public CustomDateDeserializer() { + this(null); + } + + /** + * Constructs a {@link CustomDateDeserializer} instance with the specified value type. + * + * @param t The value type to handle (can be null, handled by superclass) + */ + public CustomDateDeserializer(Class t) { + super(t); + } + + /** + * Deserializes a {@link Date} object from the given JSON parser. + * + * @param jsonParser The JSON parser containing the date value to deserialize + * @param deserializationContext The deserialization context + * @return The deserialized {@link Date} object + * @throws IOException if an I/O error occurs during deserialization + */ + @Override + public Date deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException { + + for (SimpleDateFormat formatter : dateFormatters) { + try { + return formatter.parse(jsonParser.getText()); + } catch (ParseException e) { + // ignore + } + } + + throw new IOException("Failed to deserialize a Date value."); + } +} diff --git a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java index e3513908116..38ba22db148 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java +++ b/ocaml/sdk-gen/java/autogen/xen-api/src/main/java/com/xensource/xenapi/JsonRpcClient.java @@ -32,6 +32,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; import org.apache.hc.client5.http.classic.methods.HttpPost; import org.apache.hc.client5.http.config.ConnectionConfig; import org.apache.hc.client5.http.config.RequestConfig; @@ -45,7 +46,7 @@ import java.io.IOException; import java.net.URL; -import java.text.SimpleDateFormat; +import java.util.Date; import java.util.concurrent.TimeUnit; /** @@ -209,7 +210,9 @@ protected JsonRpcResponse sendRequest(String methodCall, Object[] methodP * Helper method to initialize jackson's ObjectMapper. */ private void initializeObjectMapperConfiguration() { - this.objectMapper.setDateFormat(new SimpleDateFormat("yyyyMMdd'T'HH:mm:ss'Z'")); + var dateHandlerModule = new SimpleModule("DateHandler"); + dateHandlerModule.addDeserializer(Date.class, new CustomDateDeserializer()); + this.objectMapper.registerModule(dateHandlerModule); } /** From dc3b6d4b8c43eadfef2ba5a7e296e3a83d284921 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 8 Mar 2024 11:52:12 +0000 Subject: [PATCH 026/149] shellscripts: fix the worse complaints from shellcheck These were errors, even if they were accidentally working Signed-off-by: Pau Ruiz Safont --- ocaml/libs/http-lib/client_server_test.sh | 2 ++ ocaml/message-switch/.coverage.sh | 34 ----------------------- ocaml/sdk-gen/windows-line-endings.sh | 9 +++--- ocaml/xenopsd/scripts/vif-real | 6 ++-- scripts/install-sdk-pool | 10 +++---- scripts/test-ha-sr2 | 2 +- scripts/xe-install-supplemental-pack | 2 +- 7 files changed, 17 insertions(+), 48 deletions(-) delete mode 100644 ocaml/message-switch/.coverage.sh diff --git a/ocaml/libs/http-lib/client_server_test.sh b/ocaml/libs/http-lib/client_server_test.sh index 6757878f963..601ed257f99 100644 --- a/ocaml/libs/http-lib/client_server_test.sh +++ b/ocaml/libs/http-lib/client_server_test.sh @@ -1,3 +1,5 @@ +#!/bin/bash + set -eux trap 'kill $(jobs -p)' EXIT diff --git a/ocaml/message-switch/.coverage.sh b/ocaml/message-switch/.coverage.sh deleted file mode 100644 index 2c8f7be72b7..00000000000 --- a/ocaml/message-switch/.coverage.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash - -set -ex - -COVERAGE_DIR=.coverage -rm -rf $COVERAGE_DIR -mkdir -p $COVERAGE_DIR -pushd $COVERAGE_DIR -if [ -z "$KEEP" ]; then trap "popd; rm -rf $COVERAGE_DIR" EXIT; fi - -$(which cp) -r ../* . - -opam pin add bisect_ppx 1.3.3 -y -opam install ocveralls -y - -# install test deps -opam install message-switch-async cohttp-async -y - -export BISECT_ENABLE=YES -jbuilder runtest - -outs=($(find . | grep bisect.*.out)) -bisect-ppx-report -I $(dirname ${outs[1]}) -text report ${outs[@]} -bisect-ppx-report -I $(dirname ${outs[1]}) -summary-only -text summary ${outs[@]} -if [ -n "$HTML" ]; then bisect-ppx-report -I $(dirname ${outs[1]}) -html ../html-report ${outs[@]}; fi - -if [ -n "$TRAVIS" ]; then - echo "\$TRAVIS set; running ocveralls and sending to coveralls.io..." - ocveralls --prefix _build/default ${outs[@]} --send -else - echo "\$TRAVIS not set; displaying results of bisect-report..." - cat report - cat summary -fi diff --git a/ocaml/sdk-gen/windows-line-endings.sh b/ocaml/sdk-gen/windows-line-endings.sh index 0b11db3ba3a..f61801e1e83 100644 --- a/ocaml/sdk-gen/windows-line-endings.sh +++ b/ocaml/sdk-gen/windows-line-endings.sh @@ -1,18 +1,19 @@ +#!/bin/bash # # Copyright (c) Cloud Software Group, Inc. -# +# # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions # are met: -# +# # 1) Redistributions of source code must retain the above copyright # notice, this list of conditions and the following disclaimer. -# +# # 2) Redistributions in binary form must reproduce the above # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials # provided with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS diff --git a/ocaml/xenopsd/scripts/vif-real b/ocaml/xenopsd/scripts/vif-real index 848104ca76d..25662c9bf02 100755 --- a/ocaml/xenopsd/scripts/vif-real +++ b/ocaml/xenopsd/scripts/vif-real @@ -70,7 +70,7 @@ handle_promiscuous() if [ $? -eq 0 -a -n "${arg}" ] ; then case $NETWORK_MODE in bridge) - case "${arg}" in + case "${arg}" in true|on) echo 1 > /sys/class/net/${dev}/brport/promisc ;; *) echo 0 > /sys/class/net/${dev}/brport/promisc ;; esac @@ -198,7 +198,7 @@ PRIVATE=/xapi/${DOMUUID}/private/vif/${DEVID} HOTPLUG_STATUS="${XENBUS_PATH}/hotplug-status" HOTPLUG_ERROR="${XENBUS_PATH}/hotplug-error" -NETWORK_MODE=bridge +NETWORK_MODE=bridge if [ -e /sys/module/openvswitch ]; then NETWORK_MODE=openvswitch fi @@ -221,7 +221,7 @@ if [ -n "${UDEV_CALL}" ] && \ exit 0 fi -logger -t scripts-vif "Called as \"$@\" domid:$DOMID devid:$DEVID mode:$NETWORK_MODE" +logger -t scripts-vif "Called as \"$*\" domid:$DOMID devid:$DEVID mode:$NETWORK_MODE" case "${ACTION}" in online) if [ "${TYPE}" = "vif" ] ; then diff --git a/scripts/install-sdk-pool b/scripts/install-sdk-pool index 4cc5c6809ac..fedce55cfd4 100755 --- a/scripts/install-sdk-pool +++ b/scripts/install-sdk-pool @@ -49,7 +49,7 @@ echo "* starting VMS" echo " starting: template ${VMTEMPLATE}" ${XE} vm-start uuid=${VMTEMPLATE} -for vm in ${vms[@]} +for vm in "${vms[@]}" do VM=${vms[$i]} echo " starting: $vm" @@ -69,11 +69,11 @@ ${XEBIN} -s ${masterip} -p ${port} -u root -pw "xensource" ${NOSSL} \ host-license-add license-file="${license}" # get all ips and update a license to it -for vm in ${vms[@]} +for vm in "${vms[@]}" do ip=`${XE} vm-param-get --minimal uuid="${vm}" param-name=networks | awk '{print $2}'` echo " Applying license to ${vm} at IP (${ip})" - + ${XEBIN} -s ${ip} -p ${port} -u root -pw "xensource" ${NOSSL} \ host-license-add license-file="${license}" done @@ -82,13 +82,13 @@ sleep 40 echo "* Pooling all VMs to $masterip" -for vm in ${vms[@]} +for vm in "${vms[@]}" do ip=`${XE} vm-param-get --minimal uuid="${vm}" param-name=networks | awk '{print $2}'` echo " Pooling slave VM ${vm} to (${masterip})" ${XEBIN} -s ${ip} -p ${port} -u root -pw "xensource" ${NOSSL} \ pool-join master-address=${masterip} \ - master-username=root master-password=xensource master + master-username=root master-password=xensource master done diff --git a/scripts/test-ha-sr2 b/scripts/test-ha-sr2 index 2f35bf844ad..fc2f95980fe 100755 --- a/scripts/test-ha-sr2 +++ b/scripts/test-ha-sr2 @@ -72,7 +72,7 @@ echo -n "Waiting for Survival Rule 1: " wait_for_sr 1 echo OK -while [ /bin/true ]; do +while true; do # Block statefile everywhere remote_exec_all "iptables -I OUTPUT -p tcp --dport 3260 -j DROP" diff --git a/scripts/xe-install-supplemental-pack b/scripts/xe-install-supplemental-pack index 97f93a55188..3d830efdf9a 100755 --- a/scripts/xe-install-supplemental-pack +++ b/scripts/xe-install-supplemental-pack @@ -4,7 +4,7 @@ set -e die() { - echo $@ >&2 + echo "$@" >&2 exit 1 } From 90e4c3ca845544522c813f6d62001a72600e717e Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Fri, 15 Dec 2023 21:14:09 +0000 Subject: [PATCH 027/149] The C# project files do not need to be rendered from mustache templates any more. Signed-off-by: Konstantina Chremmou --- .../src/XenServer.csproj} | 0 ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 12 ------------ 2 files changed, 12 deletions(-) rename ocaml/sdk-gen/csharp/{templates/XenServer.csproj.mustache => autogen/src/XenServer.csproj} (100%) diff --git a/ocaml/sdk-gen/csharp/templates/XenServer.csproj.mustache b/ocaml/sdk-gen/csharp/autogen/src/XenServer.csproj similarity index 100% rename from ocaml/sdk-gen/csharp/templates/XenServer.csproj.mustache rename to ocaml/sdk-gen/csharp/autogen/src/XenServer.csproj diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index 0c70b85c6d6..d216de9e0e5 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -89,18 +89,6 @@ let rec main () = ("HTTP_actions.mustache", "HTTP_actions.cs") (gen_http_actions ()) templdir destdir ; gen_relations () ; - let sorted_members = List.sort String.compare !api_members in - let json = - `O - [ - ( "api_members" - , `A (List.map (fun x -> `O [("api_member", `String x)]) sorted_members) - ) - ] - in - render_file - ("XenServer.csproj.mustache", "XenServer.csproj") - json templdir destdir ; render_file ("ApiVersion.mustache", "ApiVersion.cs") json_releases templdir destdir From 04efef123f90c0fc251e0c1476d68e75ad4492b1 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Sat, 16 Dec 2023 00:55:59 +0000 Subject: [PATCH 028/149] Replaced the generation code of a few more files with mustache templates. Signed-off-by: Konstantina Chremmou --- ocaml/sdk-gen/csharp/gen_csharp_binding.ml | 178 +++++------- ocaml/sdk-gen/csharp/templates/Maps.mustache | 64 +++++ .../csharp/templates/Relation.mustache | 64 +++++ .../powershell/gen_powershell_binding.ml | 271 +++++------------- .../templates/ConvertTo-XenRef.mustache | 63 ++++ .../powershell/templates/HttpAction.mustache | 80 ++++++ 6 files changed, 420 insertions(+), 300 deletions(-) create mode 100644 ocaml/sdk-gen/csharp/templates/Maps.mustache create mode 100644 ocaml/sdk-gen/csharp/templates/Relation.mustache create mode 100644 ocaml/sdk-gen/powershell/templates/ConvertTo-XenRef.mustache create mode 100644 ocaml/sdk-gen/powershell/templates/HttpAction.mustache diff --git a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml index d216de9e0e5..56243c59fcd 100644 --- a/ocaml/sdk-gen/csharp/gen_csharp_binding.ml +++ b/ocaml/sdk-gen/csharp/gen_csharp_binding.ml @@ -97,33 +97,42 @@ let rec main () = and relations = Hashtbl.create 10 and gen_relations () = - let out_chan = open_out (Filename.concat destdir "Relation.cs") in - let print format = fprintf out_chan format in List.iter process_relations (relations_of_api api) ; - print - "%s\n\n\ - using System;\n\ - using System.Collections.Generic;\n\n\ - namespace XenAPI\n\ - {\n\ - \ public partial class Relation\n\ - \ {\n\ - \ public readonly String field;\n\ - \ public readonly String manyType;\n\ - \ public readonly String manyField;\n\n\ - \ public Relation(String field, String manyType, String manyField)\n\ - \ {\n\ - \ this.field = field;\n\ - \ this.manyField = manyField;\n\ - \ this.manyType = manyType;\n\ - \ }\n\n\ - \ public static Dictionary GetRelations()\n\ - \ {\n\ - \ Dictionary relations = new Dictionary();\n\n" - Licence.bsd_two_clause ; - Hashtbl.iter (gen_relations_by_type out_chan) relations ; - print "\n return relations;\n }\n }\n}\n" + let typelist = + List.rev (Hashtbl.fold (fun k v acc -> (k, v) :: acc) relations []) + in + let json = + `O + [ + ( "types" + , `A + (List.map + (fun (k, v) -> + `O + [ + ("type", `String (exposed_class_name k)) + ; ( "relations" + , `A + (List.map + (fun (x, y, z) -> + `O + [ + ("field", `String x) + ; ("manyType", `String y) + ; ("manyField", `String z) + ] + ) + v + ) + ) + ] + ) + typelist + ) + ) + ] + in + render_file ("Relation.mustache", "Relation.cs") json templdir destdir and process_relations ((oneClass, oneField), (manyClass, manyField)) = let value = @@ -132,20 +141,6 @@ and process_relations ((oneClass, oneField), (manyClass, manyField)) = in Hashtbl.replace relations manyClass value -and gen_relations_by_type out_chan manyClass relations = - let print format = fprintf out_chan format in - print " relations.Add(typeof(%s), new Relation[] {\n" - (exposed_class_name manyClass) ; - - List.iter (gen_relation out_chan) relations ; - - print " });\n\n" - -and gen_relation out_chan (manyField, oneClass, oneField) = - let print format = fprintf out_chan format in - print " new Relation(\"%s\", \"%s\", \"%s\"),\n" manyField - oneClass oneField - (* ------------------- category: http_actions *) and gen_http_actions () = (* Each action has: @@ -829,70 +824,44 @@ and gen_enum' name contents = (* ------------------- category: maps *) and gen_maps () = - let out_chan = open_out (Filename.concat destdir "Maps.cs") in - Fun.protect - (fun () -> gen_maps' out_chan) - ~finally:(fun () -> close_out out_chan) - -and gen_maps' out_chan = - let print format = fprintf out_chan format in - - print - "%s\n\n\ - using System;\n\ - using System.Collections;\n\ - using System.Collections.Generic;\n\n\ - \ namespace XenAPI\n\ - {\n\ - \ internal class Maps\n\ - \ {" Licence.bsd_two_clause ; - - TypeSet.iter (gen_map_conversion out_chan) !maps ; - - print "\n }\n}\n" - -and gen_map_conversion out_chan = function - | Map (l, r) -> - let print format = fprintf out_chan format in - let el = exposed_type l in - let el_literal = exposed_type_as_literal l in - let er = exposed_type r in - let er_literal = exposed_type_as_literal r in - - print - "\n\ - \ internal static Dictionary<%s, %s> \ - ToDictionary_%s_%s(Hashtable table)\n\ - \ {\n\ - \ Dictionary<%s, %s> result = new Dictionary<%s, %s>();\n\ - \ if (table != null)\n\ - \ {\n\ - \ foreach (string key in table.Keys)\n\ - \ {\n\ - \ try\n\ - \ {\n\ - \ %s k = %s;\n\ - \ %s v = %s;\n\ - \ result[k] = v;\n\ - \ }\n\ - \ catch\n\ - \ {\n\ - \ // continue\n\ - \ }\n\ - \ }\n\ - \ }\n\ - \ return result;\n\ - \ }\n\n" - el er - (sanitise_function_name el_literal) - (sanitise_function_name er_literal) - el er el er el - (simple_convert_from_proxy "key" l) - er - (convert_from_proxy_hashtable_value "table[key]" r) - (***) - | _ -> - assert false + let mapList = List.rev (TypeSet.fold (fun x acc -> x :: acc) !maps []) in + let json = + `O + [ + ( "all_maps" + , `A + (List.map + (function + | Map (l, r) -> + `O + [ + ("map_key", `String (exposed_type l)) + ; ("map_value", `String (exposed_type r)) + ; ( "sanitised_key" + , `String + (sanitise_function_name (exposed_type_as_literal l)) + ) + ; ( "sanitised_value" + , `String + (sanitise_function_name (exposed_type_as_literal r)) + ) + ; ( "proxy_key" + , `String (simple_convert_from_proxy "key" l) + ) + ; ( "proxy_value" + , `String + (convert_from_proxy_hashtable_value "table[key]" r) + ) + ] + | _ -> + `Null + ) + mapList + ) + ) + ] + in + render_file ("Maps.mustache", "Maps.cs") json templdir destdir (* ------------------- category: utility *) and exposed_type_opt = function @@ -979,7 +948,6 @@ and convert_from_proxy_hashtable_value thing ty = convert_from_proxy thing ty and convert_from_proxy thing ty = - (*function*) match ty with | DateTime -> thing diff --git a/ocaml/sdk-gen/csharp/templates/Maps.mustache b/ocaml/sdk-gen/csharp/templates/Maps.mustache new file mode 100644 index 00000000000..b8942e88731 --- /dev/null +++ b/ocaml/sdk-gen/csharp/templates/Maps.mustache @@ -0,0 +1,64 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +using System; +using System.Collections; +using System.Collections.Generic; + +namespace XenAPI +{ + internal class Maps + { +{{#all_maps}} + internal static Dictionary<{{{map_key}}}, {{{map_value}}}> ToDictionary_{{{sanitised_key}}}_{{{sanitised_value}}}(Hashtable table) + { + var result = new Dictionary<{{{map_key}}}, {{{map_value}}}>(); + if (table != null) + { + foreach (string key in table.Keys) + { + try + { + {{{map_key}}} k = {{{proxy_key}}}; + {{{map_value}}} v = {{{proxy_value}}}; + result[k] = v; + } + catch + { + // continue + } + } + } + return result; + } + +{{/all_maps}} + } +} diff --git a/ocaml/sdk-gen/csharp/templates/Relation.mustache b/ocaml/sdk-gen/csharp/templates/Relation.mustache new file mode 100644 index 00000000000..69f3cd8c834 --- /dev/null +++ b/ocaml/sdk-gen/csharp/templates/Relation.mustache @@ -0,0 +1,64 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +using System; +using System.Collections.Generic; + +namespace XenAPI +{ + public partial class Relation + { + public readonly String field; + public readonly String manyType; + public readonly String manyField; + + public Relation(String field, String manyType, String manyField) + { + this.field = field; + this.manyField = manyField; + this.manyType = manyType; + } + + public static Dictionary GetRelations() + { + Dictionary relations = new Dictionary(); +{{#types}} + + relations.Add(typeof({{type}}), new Relation[] { + {{#relations}} + new Relation("{{field}}", "{{manyType}}", "{{manyField}}"), + {{/relations}} + }); +{{/types}} + + return relations; + } + } +} diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 26b94fba23e..3f49c7deff8 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -18,6 +18,7 @@ module TypeSet = Set.Make (struct end) let destdir = "autogen/src" +let templdir = "templates" type cmdlet = {filename: string; content: string} @@ -67,17 +68,38 @@ let generated x = not (List.mem x.name ["blob"; "session"; "debug"; "event"; "vtpm"]) let rec main () = - gen_xenref_converters classes ; + let json = + `O + [ + ( "all_classes" + , `A + (List.map + (fun x -> + `O + [ + ("exposed_name", `String (exposed_class_name x.name)) + ; ( "var_name" + , `String (ocaml_class_to_csharp_local_var x.name) + ) + ] + ) + classes + ) + ) + ] + in + render_file + ("ConvertTo-XenRef.mustache", "ConvertTo-XenRef.cs") + json templdir destdir ; + + http_actions + |> List.filter (fun (_, (_, _, sdk, _, _, _)) -> sdk) + |> List.iter gen_http_action ; + let cmdlets = classes |> List.filter generated |> List.map gen_cmdlets |> List.concat in - let http_cmdlets = - http_actions - |> List.filter (fun (_, (_, _, sdk, _, _, _)) -> sdk) - |> List.map gen_http_action - in - let all_cmdlets = cmdlets @ http_cmdlets in - List.iter (fun x -> write_file x.filename x.content) all_cmdlets + List.iter (fun x -> write_file x.filename x.content) cmdlets (****************) (* Http actions *) @@ -87,196 +109,55 @@ and gen_http_action action = let commonVerb = get_http_action_verb name meth in let verbCategory = get_common_verb_category commonVerb in let stem = get_http_action_stem name in - let content = - sprintf - "%s\n\n\ - using System;\n\ - using System.Collections;\n\ - using System.Collections.Generic;\n\ - using System.Management.Automation;\n\ - using XenAPI;\n\n\ - namespace Citrix.XenServer.Commands\n\ - {\n\ - \ [Cmdlet(%s.%s, \"Xen%s\"%s)]\n\ - \ [OutputType(typeof(void))]\n\ - \ public class %sXen%sCommand : XenServerHttpCmdlet\n\ - \ {\n\ - \ #region Cmdlet Parameters\n\ - %s%s\n\ - \ #endregion\n\n\ - \ #region Cmdlet Methods\n\n\ - \ protected override void ProcessRecord()\n\ - \ {\n\ - \ GetSession();\n\ - %s\n\ - \ RunApiCall(() => %s);\n\ - \ }\n\n\ - \ #endregion\n\ - \ }\n\ - }\n" - Licence.bsd_two_clause verbCategory commonVerb stem - (gen_should_process_http_decl meth) - commonVerb stem - (gen_progress_tracker meth) - (gen_arg_params args) - (gen_should_process_http meth uri) - (gen_http_action_call action) + let arg_name = function + | String_query_arg x | Int64_query_arg x -> + pascal_case_rec x + | Bool_query_arg x -> + if String.lowercase_ascii x = "host" then "IsHost" else pascal_case_rec x + | Varargs_query_arg -> + "Args" in - {filename= sprintf "%s-Xen%s.cs" commonVerb stem; content} - -and gen_should_process_http_decl meth = - match meth with - | Put -> - ", SupportsShouldProcess = true" - | Get -> - ", SupportsShouldProcess = false" - | _ -> - assert false - -and gen_should_process_http meth uri = - match meth with - | Put -> - sprintf - "\n if (!ShouldProcess(\"%s\"))\n return;\n" - uri - | _ -> - "" - -and gen_progress_tracker meth = - match meth with - | Get -> - "\n\ - \ [Parameter]\n\ - \ public HTTP.DataCopiedDelegate DataCopiedDelegate { get; set; }\n" - | Put -> - "\n\ - \ [Parameter]\n\ - \ public HTTP.UpdateProgressDelegate ProgressDelegate { get; set; }\n" - | _ -> - assert false - -and gen_arg_params args = - match args with - | [] -> - "" - | hd :: tl -> - sprintf "%s%s" (gen_arg_param hd) (gen_arg_params tl) - -and gen_arg_param = function - | String_query_arg x -> - sprintf - "\n [Parameter%s]\n public string %s { get; set; }\n" - ( if String.lowercase_ascii x = "uuid" then - "(ValueFromPipelineByPropertyName = true)" - else - "" + let arg_type = function + | String_query_arg _ -> + "string" + | Int64_query_arg _ -> + "long?" + | Bool_query_arg _ -> + "bool?" + | Varargs_query_arg -> + "string[]" + in + let json = + `O + [ + ("verb_category", `String verbCategory) + ; ("common_verb", `String commonVerb) + ; ("stem", `String stem) + ; ("isPut", `Bool (meth == Put)) + ; ("isGet", `Bool (meth == Get)) + ; ("uri", `String uri) + ; ("action_name", `String name) + ; ( "args" + , `A + (List.map + (fun x -> + `O + [ + ("arg_type", `String (arg_type x)) + ; ("arg_name", `String (arg_name x)) + ; ( "from_pipeline" + , `Bool (String.lowercase_ascii (arg_name x) = "uuid") + ) + ] + ) + args + ) ) - (pascal_case_rec x) - | Int64_query_arg x -> - sprintf "\n [Parameter]\n public long? %s { get; set; }\n" - (pascal_case_rec x) - | Bool_query_arg x -> - let y = if x = "host" then "is_host" else x in - sprintf "\n [Parameter]\n public bool? %s { get; set; }\n" - (pascal_case_rec y) - | Varargs_query_arg -> - sprintf - "\n\ - \ ///

\n\ - \ /// Alternate names and values\n\ - \ ///\n\ - \ [Parameter]\n\ - \ public string[] Args { get; set; }\n" - -and gen_http_action_call (name, (meth, _, _, args, _, _)) = - let progressTracker = - match meth with - | Get -> - "DataCopiedDelegate" - | Put -> - "ProgressDelegate" - | _ -> - assert false + ] in - sprintf - "XenAPI.HTTP_actions.%s(%s,\n\ - \ CancellingDelegate, TimeoutMs, XenHost, Proxy, Path, \ - TaskRef,\n\ - \ session.opaque_ref%s)" name progressTracker - (gen_call_arg_params args) - -and gen_call_arg_params args = - match args with - | [] -> - "" - | hd :: tl -> - sprintf "%s%s" (gen_call_arg_param hd) (gen_call_arg_params tl) - -and gen_call_arg_param = function - | String_query_arg x -> - sprintf ", %s" (pascal_case_rec x) - | Int64_query_arg x -> - sprintf ", %s" (pascal_case_rec x) - | Bool_query_arg x -> - let y = if x = "host" then "is_host" else x in - sprintf ", %s" (pascal_case_rec y) - | Varargs_query_arg -> - sprintf ", Args" - -(***********************************) -(* Utility cmdlet ConvertTo-XenRef *) -(***********************************) -and gen_xenref_converters classes = - write_file "ConvertTo-XenRef.cs" (gen_body_xenref_converters classes) - -and gen_body_xenref_converters classes = - sprintf - "%s\n\n\ - using System;\n\ - using System.Collections;\n\ - using System.Collections.Generic;\n\ - using System.Management.Automation;\n\ - using XenAPI;\n\n\ - namespace Citrix.XenServer.Commands\n\ - {\n\ - \ [Cmdlet(VerbsData.ConvertTo, \"XenRef\")]\n\ - \ [OutputType(typeof(IXenObject))]\n\ - \ public class ConvertToXenRefCommand : PSCmdlet\n\ - \ {\n\ - \ #region Cmdlet Parameters\n\n\ - \ [Parameter(Mandatory = true, ValueFromPipeline = true, Position = \ - 0)]\n\ - \ public IXenObject XenObject { get; set; }\n\n\ - \ #endregion\n\n\ - \ #region Cmdlet Methods\n\n\ - \ protected override void ProcessRecord()\n\ - \ {%s\n\ - \ }\n\n\ - \ #endregion\n\n\ - \ }\n\ - }\n" - Licence.bsd_two_clause (print_converters classes) - -and print_converters classes = - match classes with - | [] -> - "" - | hd :: tl -> - sprintf - "\n\ - \ %s %s = XenObject as %s;\n\ - \ if (%s != null)\n\ - \ {\n\ - \ WriteObject(new XenRef<%s>(%s));\n\ - \ return;\n\ - \ }%s" - (qualified_class_name hd.name) - (ocaml_class_to_csharp_local_var hd.name) - (qualified_class_name hd.name) - (ocaml_class_to_csharp_local_var hd.name) - (qualified_class_name hd.name) - (ocaml_class_to_csharp_local_var hd.name) - (print_converters tl) + render_file + ("HttpAction.mustache", sprintf "%s-Xen%s.cs" commonVerb stem) + json templdir destdir (*************************) (* Autogenerated cmdlets *) diff --git a/ocaml/sdk-gen/powershell/templates/ConvertTo-XenRef.mustache b/ocaml/sdk-gen/powershell/templates/ConvertTo-XenRef.mustache new file mode 100644 index 00000000000..669704fa3d1 --- /dev/null +++ b/ocaml/sdk-gen/powershell/templates/ConvertTo-XenRef.mustache @@ -0,0 +1,63 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +using System.Management.Automation; +using XenAPI; + +namespace Citrix.XenServer.Commands +{ + [Cmdlet(VerbsData.ConvertTo, "XenRef")] + [OutputType(typeof(IXenObject))] + public class ConvertToXenRefCommand : PSCmdlet + { + #region Cmdlet Parameters + + [Parameter(Mandatory = true, ValueFromPipeline = true, Position = 0, + HelpMessage = "The API object to convert")] + public IXenObject XenObject { get; set; } + + #endregion + + #region Cmdlet Methods + + protected override void ProcessRecord() + { +{{#all_classes}} + if (XenObject is {{exposed_name}} {{var_name}}) + { + WriteObject(new XenRef<{{exposed_name}}>({{var_name}})); + return; + } +{{/all_classes}} + } + + #endregion + } +} diff --git a/ocaml/sdk-gen/powershell/templates/HttpAction.mustache b/ocaml/sdk-gen/powershell/templates/HttpAction.mustache new file mode 100644 index 00000000000..e346a68b8fe --- /dev/null +++ b/ocaml/sdk-gen/powershell/templates/HttpAction.mustache @@ -0,0 +1,80 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Management.Automation; +using XenAPI; + +namespace Citrix.XenServer.Commands +{ + [Cmdlet({{verb_category}}.{{common_verb}}, "Xen{{stem}}", SupportsShouldProcess = {{#isPut}}true{{/isPut}}{{#isGet}}false{{/isGet}})] + [OutputType(typeof(void))] + public class {{common_verb}}Xen{{stem}}Command : XenServerHttpCmdlet + { + #region Cmdlet Parameters +{{#isPut}} + + [Parameter] + public HTTP.UpdateProgressDelegate ProgressDelegate { get; set; } +{{/isPut}} +{{#isGet}} + + [Parameter] + public HTTP.DataCopiedDelegate DataCopiedDelegate { get; set; } +{{/isGet}} +{{#args}} + + [Parameter{{#from_pipeline}}(ValueFromPipelineByPropertyName = true){{/from_pipeline}}] + public {{arg_type}} {{arg_name}} { get; set; } +{{/args}} + + #endregion + + #region Cmdlet Methods + + protected override void ProcessRecord() + { + GetSession(); +{{#isPut}} + + if (!ShouldProcess("{{uri}}")) + return; +{{/isPut}} + + RunApiCall(() => HTTP_actions.{{action_name}}({{#isPut}}ProgressDelegate{{/isPut}}{{#isGet}}DataCopiedDelegate{{/isGet}}, + CancellingDelegate, TimeoutMs, XenHost, Proxy, Path, TaskRef, + session.opaque_ref{{#args}}, {{arg_name}}{{/args}})); + } + + #endregion + } +} From 64a78b4d5c05d83146860a8056f66b3d4e6a9a84 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Thu, 7 Mar 2024 15:53:10 +0100 Subject: [PATCH 029/149] GitHub Actions: Update actions/checkout to v4 to fix Node 16 warnings Signed-off-by: Bernhard Kaindl --- .github/workflows/1.249-lcm.yml | 6 +++--- .github/workflows/docs.yml | 2 +- .github/workflows/format.yml | 2 +- .github/workflows/generate-and-build-sdks.yml | 2 +- .github/workflows/hugo.yml | 2 +- .github/workflows/main.yml | 8 ++++---- .github/workflows/release.yml | 2 +- 7 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/1.249-lcm.yml b/.github/workflows/1.249-lcm.yml index 5fec8bef8d8..084750684b6 100644 --- a/.github/workflows/1.249-lcm.yml +++ b/.github/workflows/1.249-lcm.yml @@ -15,7 +15,7 @@ jobs: test: ["", "-3"] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: '1.249-lcm' @@ -30,7 +30,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: '1.249-lcm' @@ -49,7 +49,7 @@ jobs: - name: Restore opam cache id: opam-cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: "~/.opam" # invalidate cache every week, gets built using a scheduled job diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 5a05a5dfc81..0ceb3016ec7 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Pull configuration from xs-opam run: | diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index b15173805cf..60f954ffcaa 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Pull configuration from xs-opam run: | diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 92f0f52b854..d314654adde 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup XenAPI environment uses: ./.github/workflows/setup-xapi-environment diff --git a/.github/workflows/hugo.yml b/.github/workflows/hugo.yml index c76cc3d9487..2d9bfe2bcb2 100644 --- a/.github/workflows/hugo.yml +++ b/.github/workflows/hugo.yml @@ -11,7 +11,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup Hugo uses: peaceiris/actions-hugo@v2 diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d38a825a23b..9db781fd626 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,14 +21,14 @@ jobs: python-version: ["2.7", "3.11"] steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 # To check which files changed: origin/master..HEAD - uses: LizardByte/setup-python-action@master with: python-version: ${{matrix.python-version}} - - uses: actions/cache@v3 + - uses: actions/cache@v4 name: Setup cache for running pre-commit fast with: path: ~/.cache/pre-commit @@ -128,7 +128,7 @@ jobs: XAPI_VERSION: "v0.0.0" steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Setup XenAPI environment uses: ./.github/workflows/setup-xapi-environment @@ -165,7 +165,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Generate empty configuration for make to be happy run: touch config.mk diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9045949aea2..a2b07694d7a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Use python uses: actions/setup-python@v4 From 8db28fdb7345bc20e3b98f2b115e6e2e947afe96 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Thu, 7 Mar 2024 16:24:18 +0100 Subject: [PATCH 030/149] Github Actions: Update up/download-artifact to v4 to fix Node 16 warnings Signed-off-by: Bernhard Kaindl --- .github/workflows/generate-and-build-sdks.yml | 20 +++++++++---------- .github/workflows/release.yml | 12 +++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 92f0f52b854..847748e429e 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -25,13 +25,13 @@ jobs: run: opam exec -- make sdk - name: Store C# SDK source - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Source_CSharp path: _build/install/default/xapi/sdk/csharp/* - name: Store PowerShell SDK source - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Source_PowerShell path: _build/install/default/xapi/sdk/powershell/* @@ -49,7 +49,7 @@ jobs: run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve C# SDK source - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Source_CSharp path: source/ @@ -64,7 +64,7 @@ jobs: --verbosity=normal - name: Store C# SDK - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Binaries_CSharp path: source/src/bin/Release/XenServer.NET.${{ env.XAPI_VERSION_NUMBER }}-prerelease-unsigned.nupkg @@ -81,13 +81,13 @@ jobs: run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve PowerShell SDK source - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Source_PowerShell path: source/ - name: Retrieve C# SDK binaries - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_CSharp path: csharp/ @@ -135,7 +135,7 @@ jobs: ForEach-Object -Process { Copy-Item -Verbose $_.FullName -Destination "output" } - name: Store PowerShell SDK (.NET Framework 4.5) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Binaries_XenServerPowerShell_NET45 path: output/**/* @@ -155,13 +155,13 @@ jobs: run: echo "XAPI_VERSION_NUMBER=$("${{ inputs.xapi_version }}".TrimStart('v'))" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - name: Retrieve PowerShell SDK source - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Source_PowerShell path: source/ - name: Retrieve C# SDK binaries - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_CSharp path: csharp/ @@ -205,7 +205,7 @@ jobs: ForEach-Object -Process { Copy-Item -Verbose $_.FullName -Destination "output" } - name: Store PowerShell SDK (.NET ${{ matrix.dotnet }}) - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: SDK_Binaries_XenServerPowerShell_NET${{ matrix.dotnet }} path: output/**/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9045949aea2..543b3ab5a18 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -30,7 +30,7 @@ jobs: make python - name: Store python distribution artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: XenAPI path: scripts/examples/python/dist/ @@ -47,25 +47,25 @@ jobs: needs: [build-python, build-sdks] steps: - name: Retrieve Python SDK distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: XenAPI path: dist/ - name: Retrieve C# SDK distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_CSharp path: dist/ - name: Retrieve PowerShell 5.x SDK distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_XenServerPowerShell_NET45 path: sdk_powershell_5x/ - name: Retrieve PowerShell 7.x SDK distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: SDK_Binaries_XenServerPowerShell_NET6 path: sdk_powershell_7x/ @@ -96,7 +96,7 @@ jobs: id-token: write steps: - name: Retrieve python distribution artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: XenAPI path: dist/ From 38d1924413fa0b600e6b70a073a2778ed469e2b9 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Fri, 8 Mar 2024 12:00:00 +0100 Subject: [PATCH 031/149] GitHub Actions: Node 16 warning fixes: update setup-dotnet to v4 Signed-off-by: Bernhard Kaindl --- .github/workflows/generate-and-build-sdks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 92f0f52b854..eddd9f1a052 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -95,7 +95,7 @@ jobs: # Following needed for restoring packages # when calling dotnet add package - name: Set up dotnet CLI (.NET 6.0 and 8.0) - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v4 with: dotnet-version: | 6 @@ -167,7 +167,7 @@ jobs: path: csharp/ - name: Set up dotnet CLI (.NET ${{ matrix.dotnet }}) - uses: actions/setup-dotnet@v3 + uses: actions/setup-dotnet@v4 with: dotnet-version: ${{ matrix.dotnet }} From 8ec322ab01d54a994758c81f6c8c8a02d7c34f9d Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Wed, 20 Dec 2023 22:07:44 +0000 Subject: [PATCH 032/149] Split the API reference markdown into smaller files and use templates to generate it. Signed-off-by: Konstantina Chremmou --- Makefile | 11 +- ocaml/doc/basics.md | 4 + ocaml/doc/vm-lifecycle.md | 4 + ocaml/doc/wire-protocol.md | 4 + ocaml/idl/autogen/api-ref-autogen.md | 12 + ocaml/idl/autogen/dune | 6 + ocaml/idl/datamodel.ml | 14 +- ocaml/idl/datamodel_errors.ml | 2 +- ocaml/idl/datamodel_host.ml | 2 +- ocaml/idl/datamodel_main.ml | 2 +- ocaml/idl/dune | 11 + ocaml/idl/markdown_backend.ml | 815 ++++++++++----------- ocaml/idl/templates/api_errors.mustache | 136 ++++ ocaml/idl/templates/class.mustache | 89 +++ ocaml/idl/templates/classes.mustache | 13 + ocaml/idl/templates/relationships.mustache | 19 + ocaml/idl/templates/toc.mustache | 15 + ocaml/idl/templates/types.mustache | 40 + 18 files changed, 749 insertions(+), 450 deletions(-) create mode 100644 ocaml/idl/autogen/api-ref-autogen.md create mode 100644 ocaml/idl/autogen/dune create mode 100644 ocaml/idl/templates/api_errors.mustache create mode 100644 ocaml/idl/templates/class.mustache create mode 100644 ocaml/idl/templates/classes.mustache create mode 100644 ocaml/idl/templates/relationships.mustache create mode 100644 ocaml/idl/templates/toc.mustache create mode 100644 ocaml/idl/templates/types.mustache diff --git a/Makefile b/Makefile index f272cd8e766..991ce87c812 100644 --- a/Makefile +++ b/Makefile @@ -74,15 +74,20 @@ schema: dune runtest ocaml/idl doc: - dune build --profile=$(PROFILE) ocaml/idl/datamodel_main.exe +#html dune build --profile=$(PROFILE) -f @ocaml/doc/jsapigen mkdir -p $(XAPIDOC)/html cp -r _build/default/ocaml/doc/api $(XAPIDOC)/html cp _build/default/ocaml/doc/branding.js $(XAPIDOC)/html cp ocaml/doc/*.js ocaml/doc/*.html ocaml/doc/*.css $(XAPIDOC)/html - dune exec --profile=$(PROFILE) -- ocaml/idl/datamodel_main.exe -closed -markdown $(XAPIDOC)/markdown - cp ocaml/doc/*.dot ocaml/doc/doc-convert.sh $(XAPIDOC) +#markdown + dune build --profile=$(PROFILE) -f @ocaml/idl/markdowngen + mkdir -p $(XAPIDOC)/markdown + cp -r _build/default/ocaml/idl/autogen/*.md $(XAPIDOC)/markdown + cp -r _build/default/ocaml/idl/autogen/*.yml $(XAPIDOC)/markdown find ocaml/doc -name "*.md" -not -name "README.md" -exec cp {} $(XAPIDOC)/markdown/ \; +#other + cp ocaml/doc/*.dot ocaml/doc/doc-convert.sh $(XAPIDOC) # Build manpages, networkd generated these dune build --profile=$(PROFILE) -f @man diff --git a/ocaml/doc/basics.md b/ocaml/doc/basics.md index f659a795a14..9a75a16087b 100644 --- a/ocaml/doc/basics.md +++ b/ocaml/doc/basics.md @@ -1,3 +1,7 @@ +--- + layout: doc +--- + # API Basics This document defines the XenServer Management API - an interface for remotely diff --git a/ocaml/doc/vm-lifecycle.md b/ocaml/doc/vm-lifecycle.md index 31f31889f36..68664730617 100644 --- a/ocaml/doc/vm-lifecycle.md +++ b/ocaml/doc/vm-lifecycle.md @@ -1,3 +1,7 @@ +--- + layout: doc +--- + # VM Lifecycle The following diagram shows the states that a VM can be in diff --git a/ocaml/doc/wire-protocol.md b/ocaml/doc/wire-protocol.md index 260039ab495..e42a0f2da2c 100644 --- a/ocaml/doc/wire-protocol.md +++ b/ocaml/doc/wire-protocol.md @@ -1,3 +1,7 @@ +--- + layout: doc +--- + # Wire Protocol for Remote API Calls API calls are sent over a network to a Xen-enabled host using an RPC protocol. diff --git a/ocaml/idl/autogen/api-ref-autogen.md b/ocaml/idl/autogen/api-ref-autogen.md new file mode 100644 index 00000000000..7abf22b6872 --- /dev/null +++ b/ocaml/idl/autogen/api-ref-autogen.md @@ -0,0 +1,12 @@ +--- + layout: doc +--- + +# API Reference + +Version **@xapi-version@** + +- [Classes](@root@management-api/classes.html) +- [Relationships Between Classes](@root@management-api/relationships-between-classes.html) +- [Types](@root@management-api/types.html) +- [ErrorHandling](@root@management-api/api-ref-autogen-errors.html) diff --git a/ocaml/idl/autogen/dune b/ocaml/idl/autogen/dune new file mode 100644 index 00000000000..483a0dbdef8 --- /dev/null +++ b/ocaml/idl/autogen/dune @@ -0,0 +1,6 @@ +(alias + (name markdowngen) + (deps + (source_tree .) + ) +) \ No newline at end of file diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index 15f1c4c66c6..d7bff0266a9 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -5985,7 +5985,7 @@ module DR_task = struct ) ; (Set String, "whitelist", "The devices to use for disaster recovery") ] - ~result:(Ref _dr_task, "The reference to the created task") + ~result:(Ref _dr_task, "The reference of the created DR_task") ~doc: "Create a disaster recovery task which will query the supplied list of \ devices" @@ -6202,7 +6202,7 @@ module Blob = struct } ] ~doc:"Create a placeholder for a binary blob" ~flags:[`Session] - ~result:(Ref _blob, "The reference to the created blob") + ~result:(Ref _blob, "The reference of the created blob") ~allowed_roles:_R_POOL_OP () let destroy = @@ -6889,7 +6889,8 @@ module GPU_group = struct ; param_default= Some (VMap []) } ] - ~result:(Ref _gpu_group, "") ~allowed_roles:_R_POOL_OP () + ~result:(Ref _gpu_group, "The reference of the created GPU_group") + ~allowed_roles:_R_POOL_OP () let destroy = call ~name:"destroy" @@ -7041,7 +7042,7 @@ module VGPU = struct ; param_default= Some (VRef null_ref) } ] - ~result:(Ref _vgpu, "reference to the newly created object") + ~result:(Ref _vgpu, "The reference of the created VGPU object") ~allowed_roles:_R_POOL_OP () let destroy = @@ -7356,7 +7357,7 @@ module PVS_proxy = struct let create = call ~name:"create" ~doc:"Configure a VM/VIF to use a PVS proxy" - ~result:(Ref _pvs_proxy, "the new PVS proxy") + ~result:(Ref _pvs_proxy, "The reference of the created PVS proxy") ~params: [ (Ref _pvs_site, "site", "PVS site that we proxy for") @@ -7626,7 +7627,8 @@ module USB_group = struct ; param_default= Some (VMap []) } ] - ~result:(Ref _usb_group, "") ~allowed_roles:_R_POOL_ADMIN () + ~result:(Ref _usb_group, "The reference of the created USB_group") + ~allowed_roles:_R_POOL_ADMIN () let destroy = call ~name:"destroy" ~lifecycle diff --git a/ocaml/idl/datamodel_errors.ml b/ocaml/idl/datamodel_errors.ml index 0a0166dfe93..0bfbaa21039 100644 --- a/ocaml/idl/datamodel_errors.ml +++ b/ocaml/idl/datamodel_errors.ml @@ -1455,7 +1455,7 @@ let _ = ~doc: "The requested update could not be found. Please upload the update \ again. This can occur when you run xe update-pool-clean before xe \ - update-apply. " + update-apply." () ; error Api_errors.update_pool_apply_failed ["hosts"] ~doc:"The update cannot be applied for the following host(s)." () ; diff --git a/ocaml/idl/datamodel_host.ml b/ocaml/idl/datamodel_host.ml index 6c7895ec901..2f9d1d7ed83 100644 --- a/ocaml/idl/datamodel_host.ml +++ b/ocaml/idl/datamodel_host.ml @@ -1369,7 +1369,7 @@ let set_power_on_mode = ; (Changed, rel_stockholm, "Removed iLO script") ] ~in_product_since:rel_midnight_ride - ~doc:"Set the power-on-mode, host, user and password " + ~doc:"Set the power-on-mode, host, user and password" ~params: [ (Ref _host, "self", "The host") diff --git a/ocaml/idl/datamodel_main.ml b/ocaml/idl/datamodel_main.ml index 77250738817..fa22d3b9d09 100644 --- a/ocaml/idl/datamodel_main.ml +++ b/ocaml/idl/datamodel_main.ml @@ -86,7 +86,7 @@ let _ = in if !markdown_mode then - Markdown_backend.all api !dirname ; + Markdown_backend.all api ; if !dirname <> "" then Unix.chdir !dirname ; if !dot_mode then diff --git a/ocaml/idl/dune b/ocaml/idl/dune index 3dfa75af8c4..713462e7ffa 100644 --- a/ocaml/idl/dune +++ b/ocaml/idl/dune @@ -30,6 +30,7 @@ (modules datamodel_main dot_backend dtd_backend markdown_backend) (libraries dune-build-info + mustache xapi-datamodel xapi-stdext-std xapi-stdext-pervasives @@ -37,6 +38,16 @@ ) ) +(rule + (alias markdowngen) + (deps + (:x datamodel_main.exe) + (source_tree templates) + ) + (package xapi-datamodel) + (action (run %{x} -closed -markdown)) +) + (test (name schematest) (modes exe) diff --git a/ocaml/idl/markdown_backend.ml b/ocaml/idl/markdown_backend.ml index edd95d95d50..c4bbd538fe1 100644 --- a/ocaml/idl/markdown_backend.ml +++ b/ocaml/idl/markdown_backend.ml @@ -15,7 +15,7 @@ open Printf open Datamodel_types open Datamodel_utils open Dm_api -open Xapi_stdext_pervasives.Pervasiveext +module Unixext = Xapi_stdext_unix.Unixext (*column widths for the autogenerated tables*) let col_width_15 = 15 @@ -28,6 +28,10 @@ let col_width_40 = 40 let col_width_70 = 70 +let destdir = "autogen" + +let templatesdir = "templates" + let pad_right x max_width = let length = String.length x in if String.length x < max_width then @@ -78,14 +82,6 @@ let escape s = let escaped_list = List.map esc_char sl in String.concat "" escaped_list -let is_prim_type = function - | String | Int | Float | Bool | DateTime -> - true - | _ -> - false - -let is_prim_opt_type = function None -> true | Some (ty, _) -> is_prim_type ty - let rec of_ty_verbatim = function | SecretString | String -> "string" @@ -152,231 +148,383 @@ let string_of_qualifier = function | RW -> "_RW_" -let is_removal_marker x = - match x with Lifecycle.Removed, _, _ -> true | _ -> false +let render_file (infile, outfile) json templates_dir dest_dir = + let templ = + Unixext.string_of_file (Filename.concat templates_dir infile) + |> Mustache.of_string + in + let rendered = Mustache.render templ json in + let io = open_out (Filename.concat dest_dir outfile) in + Fun.protect + (fun () -> output_string io rendered) + ~finally:(fun () -> close_out io) -let is_deprecation_marker x = - match x with Lifecycle.Deprecated, _, _ -> true | _ -> false +let generate_class cls = + let class_json = + `O + [ + ("class_name", `String (escape cls.name)) + ; ("class_descr", `String (escape cls.description)) + ; ("has_descr", `Bool (cls.description <> "")) + ; ("class_deprecated", `Bool (cls.obj_lifecycle.state = Deprecated_s)) + ; ("class_removed", `Bool (cls.obj_lifecycle.state = Removed_s)) + ; ("is_event", `Bool (String.lowercase_ascii cls.name = "event")) + ; ("has_fields", `Bool (Datamodel_utils.fields_of_obj cls <> [])) + ; ( "fields" + , `A + (cls + |> Datamodel_utils.fields_of_obj + |> List.sort (fun x y -> + compare_case_ins + (Datamodel_utils.wire_name_of_field x) + (Datamodel_utils.wire_name_of_field y) + ) + |> List.map (fun field -> + `O + [ + ( "field_name" + , `String + (pad_right + (escape (Datamodel_utils.wire_name_of_field field)) + col_width_20 + ) + ) + ; ( "field_type" + , `String + (pad_right + ("`" ^ of_ty_verbatim field.ty ^ "`") + col_width_20 + ) + ) + ; ( "field_ctor" + , `String + (pad_right + (string_of_qualifier field.qualifier) + col_width_15 + ) + ) + ; ( "field_descr" + , `String + (pad_right + (escape field.field_description) + col_width_40 + ) + ) + ; ( "field_deprecated" + , `Bool + (field.lifecycle.state = Deprecated_s + || cls.obj_lifecycle.state = Deprecated_s + ) + ) + ; ( "field_removed" + , `Bool + (field.lifecycle.state = Removed_s + || cls.obj_lifecycle.state = Removed_s + ) + ) + ] + ) + ) + ) + ; ("has_rpcs", `Bool (cls.messages <> [])) + ; ( "all_rpcs" + , `A + (cls.messages + |> List.sort (fun x y -> compare_case_ins x.msg_name y.msg_name) + |> List.map (fun msg -> + let is_event_from = + String.lowercase_ascii cls.name = "event" + && String.lowercase_ascii msg.msg_name = "from" + in + let rpc_param_csv = + msg.msg_params + |> List.map (fun p -> + of_ty_verbatim p.param_type ^ " " ^ p.param_name + ) + |> String.concat ", " + in + let error_codes_csv = + msg.msg_errors + |> List.map (fun x -> sprintf "`%s`" x.err_name) + |> String.concat ", " + in + let rbac x = + match x.msg_allowed_roles with + | Some y when y <> [] -> + List.hd (List.rev y) + | _ -> + "" + in + `O + [ + ("rpc_name_escaped", `String (escape msg.msg_name)) + ; ("rpc_name", `String msg.msg_name) + ; ("rpc_descr", `String (escape msg.msg_doc)) + ; ("rpc_has_descr", `Bool (msg.msg_doc <> "")) + ; ( "rpc_deprecated" + , `Bool + (msg.msg_lifecycle.state = Lifecycle.Deprecated_s + || cls.obj_lifecycle.state = Deprecated_s + ) + ) + ; ( "rpc_removed" + , `Bool + (msg.msg_lifecycle.state = Lifecycle.Removed_s + || cls.obj_lifecycle.state = Removed_s + ) + ) + ; ("returns_void", `Bool (msg.msg_result = None)) + ; ( "return_type" + , `String + ( if is_event_from then + "event batch" + else + of_ty_opt_verbatim msg.msg_result + ) + ) + ; ( "return_descr" + , `String (escape (desc_of_ty_opt msg.msg_result)) + ) + ; ("rpc_param_csv", `String rpc_param_csv) + ; ("has_rbac", `Bool (rbac msg <> "")) + ; ("min_role", `String (rbac msg)) + ; ("session", `Bool msg.msg_session) + ; ("has_rpc_params", `Bool (msg.msg_params <> [])) + ; ( "rpc_params" + , `A + (msg.msg_params + |> List.map (fun p -> + `O + [ + ( "param_name" + , `String + (pad_right (escape p.param_name) + col_width_30 + ) + ) + ; ( "param_type" + , `String + (pad_right + ("`" + ^ of_ty_verbatim p.param_type + ^ "`" + ) + col_width_30 + ) + ) + ; ( "param_descr" + , `String + (pad_right (escape p.param_doc) + col_width_40 + ) + ) + ] + ) + ) + ) + ; ("has_error_codes", `Bool (msg.msg_errors <> [])) + ; ("error_codes_csv", `String error_codes_csv) + ] + ) + ) + ) + ] + in + render_file + ("class.mustache", sprintf "class-%s.md" (String.lowercase_ascii cls.name)) + class_json templatesdir destdir -(* Make a markdown section for an API-specified message *) -let markdown_section_of_message printer obj ~is_class_deprecated - ~is_class_removed x = - let is_event_from = - String.lowercase_ascii obj.name = "event" - && String.lowercase_ascii x.msg_name = "from" +let generate_types system = + let type_comparer x y = + match (x, y) with + | Enum (a, _), Enum (b, _) -> + compare_case_ins a b + | _ -> + compare x y in - let return_type = of_ty_opt_verbatim x.msg_result in - printer (sprintf "#### RPC name: %s" (escape x.msg_name)) ; - printer "" ; - if x.msg_lifecycle.state = Lifecycle.Removed_s || is_class_removed then ( - printer "**This message is removed.**" ; - printer "" - ) else if - x.msg_lifecycle.state = Lifecycle.Deprecated_s || is_class_deprecated - then ( - printer "**This message is deprecated.**" ; - printer "" - ) ; - printer "_Overview:_" ; - printer "" ; - printer (escape x.msg_doc) ; - printer "" ; - printer "_Signature:_" ; - printer "" ; - printer "```" ; - let result = - if is_event_from then - "" - else - of_ty_opt_verbatim x.msg_result + let enums = + Types.of_objects system + |> List.filter (function Enum (_, _) -> true | _ -> false) + |> List.sort type_comparer in - printer - (sprintf "%s %s (%s)" result x.msg_name - (String.concat ", " - ((if x.msg_session then ["session ref session_id"] else []) - @ List.map - (fun p -> of_ty_verbatim p.param_type ^ " " ^ p.param_name) - x.msg_params - ) - ) - ) ; - printer "```" ; - printer "" ; - if x.msg_params <> [] then ( - printer "_Arguments:_" ; - printer "" ; - printer - "|type |name \ - |description |" ; - printer - "|:-----------------------------|:-----------------------------|:---------------------------------------|" ; - if x.msg_session then - printer - "|session ref |session_id \ - |Reference to a valid session |" ; - let get_param_row p = - sprintf "|`%s`|%s|%s|" - (pad_right (of_ty_verbatim p.param_type) (col_width_30 - 2)) - (pad_right (escape p.param_name) col_width_30) - (pad_right (escape p.param_doc) col_width_40) - in - List.iter (fun p -> printer (get_param_row p)) x.msg_params ; - printer "" - ) ; - let print_rbac y = - match y.msg_allowed_roles with - | Some yy when yy <> [] -> - printer ("_Minimum Role:_ " ^ List.hd (List.rev yy)) ; - printer "" - | _ -> - () + let types_json = + `O + [ + ( "enums" + , `A + (List.map + (function + | Enum (name, options) -> + `O + [ + ("enum", `String (pad_right name (col_width_40 - 5))) + ; ( "enum_options" + , `A + (options + |> List.sort (fun (x, _) (y, _) -> + compare_case_ins x y + ) + |> List.map (fun (n, c) -> + `O + [ + ( "option_name" + , `String + (pad_right + ("`" ^ n ^ "`") + col_width_40 + ) + ) + ; ( "option_descr" + , `String + (pad_right (escape c) col_width_40) + ) + ] + ) + ) + ) + ] + | _ -> + `Null + ) + enums + ) + ) + ] in - print_rbac x ; - printer - ("_Return Type:_" - ^ if is_event_from then " an event batch" else sprintf " `%s`" return_type - ) ; - printer "" ; - let descr = desc_of_ty_opt x.msg_result in - if descr <> "" then ( - printer (escape descr) ; - printer "" - ) ; - if x.msg_errors <> [] then ( - let error_codes = - List.map (fun err -> sprintf "`%s`" err.err_name) x.msg_errors - in - printer - (sprintf "_Possible Error Codes:_ %s" (String.concat ", " error_codes)) ; - printer "" - ) + render_file ("types.mustache", "types.md") types_json templatesdir destdir -let print_field_table_of_obj printer ~is_class_deprecated ~is_class_removed x = - printer (sprintf "### Fields for class: " ^ escape x.name) ; - printer "" ; - if x.contents = [] then - printer ("Class " ^ escape x.name ^ " has no fields.") - else ( - printer - "|Field |Type |Qualifier \ - |Description |" ; - printer - "|:-------------------|:-------------------|:--------------|:---------------------------------------|" ; - let print_field_content printer - ({qualifier; ty; field_description= description; _} as y) = - let wired_name = Datamodel_utils.wire_name_of_field y in - let descr = - ( if y.lifecycle.state = Removed_s || is_class_removed then - "**Removed**. " - else if y.lifecycle.state = Deprecated_s || is_class_deprecated then - "**Deprecated**. " - else - "" - ) - ^ escape description - in - printer - (sprintf "|%s|`%s`|%s|%s|" - (pad_right (escape wired_name) col_width_20) - (pad_right (of_ty_verbatim ty) (col_width_20 - 2)) - (pad_right (string_of_qualifier qualifier) col_width_15) - (pad_right descr col_width_40) - ) - in - x - |> Datamodel_utils.fields_of_obj - |> List.sort (fun x y -> - compare_case_ins - (Datamodel_utils.wire_name_of_field x) - (Datamodel_utils.wire_name_of_field y) - ) - |> List.iter (print_field_content printer) ; - if String.lowercase_ascii x.name = "event" then - printer - (sprintf "|%s|`%s`|%s|%s|" - (pad_right "snapshot" col_width_20) - (pad_right "" (col_width_20 - 2)) - (pad_right "_RO/runtime_" col_width_15) - (pad_right - "The record of the database object that was added, changed or \ - deleted" - col_width_40 - ) +let generate_relationships api = + let relations = relations_of_api api in + let relationships_json = + `O + [ + ( "relationships" + , `A + (List.map + (function + | ((a, a_field), (b, b_field)) as rel -> + let c = Relations.classify api rel in + let afield = "`" ^ a ^ "." ^ a_field ^ "`" in + let bfield = "`" ^ b ^ "." ^ b_field ^ "`" in + `O + [ + ( "a_field" + , `String (pad_right afield (col_width_40 - 2)) + ) + ; ( "b_field" + , `String (pad_right bfield (col_width_40 - 2)) + ) + ; ( "relationship" + , `String + (pad_right + (Relations.string_of_classification c) + col_width_15 + ) + ) + ] + ) + relations + ) ) - ) + ] + in + render_file + ("relationships.mustache", "relationships-between-classes.md") + relationships_json templatesdir destdir -let of_obj printer x = - printer (sprintf "## Class: %s" (escape x.name)) ; - printer "" ; - let is_class_removed = x.obj_lifecycle.state = Removed_s in - let is_class_deprecated = x.obj_lifecycle.state = Deprecated_s in - if is_class_removed then ( - printer "**This class is removed.**" ; - printer "" - ) else if is_class_deprecated then ( - printer "**This class is deprecated.**" ; - printer "" - ) ; - printer (escape x.description) ; - printer "" ; - print_field_table_of_obj printer ~is_class_deprecated ~is_class_removed x ; - printer "" ; - printer (sprintf "### RPCs associated with class: " ^ escape x.name) ; - printer "" ; - if x.messages = [] then ( - printer - (sprintf "Class %s has no additional RPCs associated with it." - (escape x.name) - ) ; - printer "" - ) else - x.messages - |> List.sort (fun x y -> compare_case_ins x.msg_name y.msg_name) - |> List.iter - (markdown_section_of_message printer x ~is_class_deprecated - ~is_class_removed - ) +let generate_classes system = + let classes_json = + `O + [ + ( "classes" + , `A + (List.map + (fun x -> + let notice y = + match y.obj_lifecycle.state with + | Removed_s -> + "**Removed**. " + | Deprecated_s -> + "**Deprecated**. " + | _ -> + "" + in + `O + [ + ("name", `String x.name) + ; ("name_lower", `String (String.lowercase_ascii x.name)) + ; ( "description" + , `String + (pad_right + (notice x ^ escape x.description) + col_width_70 + ) + ) + ] + ) + system + ) + ) + ] + in + render_file + ("classes.mustache", "classes.md") + classes_json templatesdir destdir -let print_enum printer = function - | Enum (name, options) -> - printer - (sprintf "|`enum %s`| |" - (pad_right name (col_width_40 - 7)) - ) ; - printer - "|:---------------------------------------|:---------------------------------------|" ; - let print_option (opt, description) = - printer - (sprintf "|`%s`|%s|" - (pad_right opt (col_width_40 - 2)) - (pad_right (escape description) col_width_40) - ) - in - options - |> List.sort (fun (x, _) (y, _) -> compare_case_ins x y) - |> List.iter print_option ; - printer "" - | _ -> - () +let generate_toc system = + let classes_json = + `O + [ + ( "classes" + , `A + (List.map + (fun x -> + `O + [ + ("name", `String x.name) + ; ("name_lower", `String (String.lowercase_ascii x.name)) + ] + ) + system + ) + ) + ] + in + render_file ("toc.mustache", "toc.yml") classes_json templatesdir destdir -let error_doc printer {err_name= name; err_params= params; err_doc= doc} = - printer (sprintf "### %s" (escape name)) ; - printer "" ; - printer (escape doc) ; - printer "" ; - if params = [] then - printer "No parameters." - else ( - printer "_Signature:_" ; - printer "" ; - printer "```" ; - printer (sprintf "%s(%s)" name (String.concat ", " params)) ; - printer "```" - ) ; - printer "" +let generate_errors () = + (* Sort the errors alphabetically, then generate one section per code. *) + let errs = + Hashtbl.fold (fun name err acc -> (name, err) :: acc) Datamodel.errors [] + |> List.sort (fun (n1, _) (n2, _) -> compare n1 n2) + |> List.split + |> snd + in + let error_json = + `O + [ + ( "errors" + , `A + (List.map + (fun {err_name; err_params; err_doc} -> + `O + [ + ("error_code", `String (escape err_name)) + ; ("error_code_unescaped", `String err_name) + ; ("error_description", `String (escape err_doc)) + ; ("parameters", `String (String.concat ", " err_params)) + ] + ) + errs + ) + ) + ] + in + render_file + ("api_errors.mustache", "api-ref-autogen-errors.md") + error_json templatesdir destdir -let print_classes api io = - let printer text = fprintf io "%s\n" text in +let all api = (* Remove private messages that are only used internally (e.g. get_record_internal) *) let api = Dm_api.filter @@ -390,219 +538,10 @@ let print_classes api io = let system = objects_of_api api |> List.sort (fun x y -> compare_case_ins x.name y.name) in - let relations = relations_of_api api in - printer - "# API Reference - Types and Classes\n\n\ - ## Classes\n\n\ - The following classes are defined:\n\n\ - |Name \ - |Description |\n\ - |:-------------------|:---------------------------------------------------------------------|" ; - let get_descr obj = - ( if obj.obj_lifecycle.state = Removed_s then - "**Removed**. " - else if obj.obj_lifecycle.state = Deprecated_s then - "**Deprecated**. " - else - "" - ) - ^ escape obj.description - in - List.iter - (fun obj -> - printer - (sprintf "|`%s`|%s|" - (pad_right obj.name (col_width_20 - 2)) - (pad_right (get_descr obj) col_width_70) - ) - ) - system ; - printer - "\n\ - ## Relationships Between Classes\n\n\ - Fields that are bound together are shown in the following table:\n\n\ - |_object.field_ \ - |_object.field_ |_relationship_ |\n\ - |:---------------------------------------|:---------------------------------------|:--------------|" ; - List.iter - (function - | ((a, a_field), (b, b_field)) as rel -> - let c = Relations.classify api rel in - let afield = a ^ "." ^ a_field in - let bfield = b ^ "." ^ b_field in - printer - (sprintf "|`%s`|`%s`|%s|" - (pad_right afield (col_width_40 - 2)) - (pad_right bfield (col_width_40 - 2)) - (pad_right (Relations.string_of_classification c) col_width_15) - ) - ) - relations ; - printer - "\n\ - The following figure represents bound fields (as specified above) \ - diagramatically, using crow's foot notation to specify one-to-one, \ - one-to-many or many-to-many relationships:\n\n\ - ![Class relationships](classes.png 'Class relationships')\n\n\ - ## Types\n\n\ - ### Primitives\n\n\ - The following primitive types are used to specify methods and fields in \ - the API Reference:\n\n\ - |Type |Description |\n\ - |:-------|:-------------------------------------------|\n\ - |string |text strings |\n\ - |int |64-bit integers |\n\ - |float |IEEE double-precision floating-point numbers|\n\ - |bool |boolean |\n\ - |datetime|date and timestamp |\n\n\ - ### Higher-order types\n\n\ - The following type constructors are used:\n\n\ - |Type \ - |Description |\n\ - |:-----------------|:-------------------------------------------------------|\n\ - |_c_ ref |reference to an object of class \ - _c_ |\n\ - |_t_ set |a set of elements of type \ - _t_ |\n\ - |(_a -> b_) map |a table mapping values of type _a_ to values \ - of type _b_|\n\n\ - ### Enumeration types\n\n\ - The following enumeration types are used:\n" ; - let type_comparer x y = - match (x, y) with - | Enum (a, _), Enum (b, _) -> - compare_case_ins a b - | _ -> - compare x y - in - Types.of_objects system - |> List.sort type_comparer - |> List.iter (print_enum printer) ; - List.iter (fun x -> of_obj printer x) system -let print_errors io = - let printer text = fprintf io "%s\n" text in - printer - "# API Reference - Error Handling\n\n\ - When a low-level transport error occurs, or a request is malformed at the \ - HTTP\n\ - or RPC level, the server may send an HTTP 500 error response, or the client\n\ - may simulate the same. The client must be prepared to handle these errors,\n\ - though they may be treated as fatal.\n\n\ - On the wire, these are transmitted in a form similar to this when using the\n\ - XML-RPC protocol:\n\n\ - ```\n\ - $curl -D - -X POST https://server -H 'Content-Type: application/xml' \\\n\ - > -d '\n\ - > \n\ - > session.logout\n\ - > '\n\ - HTTP/1.1 500 Internal Error\n\ - content-length: 297\n\ - content-type:text/html\n\ - connection:close\n\ - cache-control:no-cache, no-store\n\n\ -

HTTP 500 internal server error

An unexpected error \ - occurred;\n\ - \ please wait a while and try again. If the problem persists, please \ - contact your\n\ - \ support representative.

Additional information \ -

Xmlrpc.Parse_error(&quo\n\ - t;close_tag", "open_tag", _)\n\ - ```\n\n\ - When using the JSON-RPC protocol:\n\n\ - ```\n\ - $curl -D - -X POST https://server/jsonrpc -H 'Content-Type: \ - application/json' \\\n\ - > -d '{\n\ - > \"jsonrpc\": \"2.0\",\n\ - > \"method\": \"session.login_with_password\",\n\ - > \"id\": 0\n\ - > }'\n\ - HTTP/1.1 500 Internal Error\n\ - content-length: 308\n\ - content-type:text/html\n\ - connection:close\n\ - cache-control:no-cache, no-store\n\n\ -

HTTP 500 internal server error

An unexpected error \ - occurred;\n\ - \ please wait a while and try again. If the problem persists, please \ - contact your\n\ - \ support representative.

Additional information \ -

Jsonrpc.Malformed_metho\n\ - d_request("{jsonrpc=...,method=...,id=...}")\n\ - ```\n\n\ - All other failures are reported with a more structured error response, to\n\ - allow better automatic response to failures, proper internationalisation of\n\ - any error message, and easier debugging.\n\n\ - On the wire, these are transmitted like this when using the XML-RPC \ - protocol:\n\n\ - ```xml\n\ - \ \n\ - \ \n\ - \ Status\n\ - \ Failure\n\ - \ \n\ - \ \n\ - \ ErrorDescription\n\ - \ \n\ - \ \n\ - \ \n\ - \ MAP_DUPLICATE_KEY\n\ - \ Customer\n\ - \ eSpiel Inc.\n\ - \ eSpiel Incorporated\n\ - \ \n\ - \ \n\ - \ \n\ - \ \n\ - \ \n\ - ```\n\n\ - Note that `ErrorDescription` value is an array of string values. The\n\ - first element of the array is an error code; the remainder of the array are\n\ - strings representing error parameters relating to that code. In this case,\n\ - the client has attempted to add the mapping _Customer ->\n\ - eSpiel Incorporated_ to a Map, but it already contains the mapping\n\ - _Customer -> eSpiel Inc._, and so the request has failed.\n\n\ - When using the JSON-RPC protocol v2.0, the above error is transmitted as:\n\n\ - ```json\n\ - {\n\ - \ \"jsonrpc\": \"2.0\",\n\ - \ \"error\": {\n\ - \ \"code\": 1,\n\ - \ \"message\": \"MAP_DUPLICATE_KEY\",\n\ - \ \"data\": [\n\ - \ \"Customer\",\"eSpiel Inc.\",\"eSpiel Incorporated\"\n\ - \ ]\n\ - \ },\n\ - \ \"id\": 3\n\ - \ }\n\ - ```\n\n\ - Finally, when using the JSON-RPC protocol v1.0:\n\n\ - ```json\n\ - {\n\ - \ \"result\": null,\n\ - \ \"error\": [\n\ - \ \"MAP_DUPLICATE_KEY\",\"Customer\",\"eSpiel Inc.\",\"eSpiel \ - Incorporated\"\n\ - \ ],\n\ - \ \"id\": \"xyz\"\n\ - }\n\ - ```\n\n\ - Each possible error code is documented in the following section.\n\n\ - ## Error Codes\n" ; - (* Sort the errors alphabetically, then generate one section per code. *) - let errs = - Hashtbl.fold (fun name err acc -> (name, err) :: acc) Datamodel.errors [] - in - List.iter (error_doc printer) - (snd (List.split (List.sort (fun (n1, _) (n2, _) -> compare n1 n2) errs))) - -let all api destdir = - Xapi_stdext_unix.Unixext.mkdir_rec destdir 0o755 ; - let with_file filename f = - let io = open_out (Filename.concat destdir filename) in - finally (fun () -> f io) (fun () -> close_out io) - in - with_file "api-ref-autogen.md" (print_classes api) ; - with_file "api-ref-autogen-errors.md" print_errors + List.iter generate_class system ; + generate_classes system ; + generate_relationships api ; + generate_types system ; + generate_errors () ; + generate_toc system diff --git a/ocaml/idl/templates/api_errors.mustache b/ocaml/idl/templates/api_errors.mustache new file mode 100644 index 00000000000..e1a4b95fbaa --- /dev/null +++ b/ocaml/idl/templates/api_errors.mustache @@ -0,0 +1,136 @@ +--- + layout: doc +--- + +# Error Handling + +When a low-level transport error occurs, or a request is malformed at the HTTP +or RPC level, the server may send an HTTP 500 error response, or the client +may simulate the same. The client must be prepared to handle these errors, +though they may be treated as fatal. + +On the wire, these are transmitted in a form similar to this when using the +XML-RPC protocol: + +``` +$curl -D - -X POST https://server -H 'Content-Type: application/xml' \ +> -d ' +> +> session.logout +> ' +HTTP/1.1 500 Internal Error +content-length: 297 +content-type:text/html +connection:close +cache-control:no-cache, no-store + +

HTTP 500 internal server error

An unexpected error occurred; + please wait a while and try again. If the problem persists, please contact your + support representative.

Additional information

Xmlrpc.Parse_error(&quo +t;close_tag", "open_tag", _) +``` + +When using the JSON-RPC protocol: + +``` +$curl -D - -X POST https://server/jsonrpc -H 'Content-Type: application/json' \ +> -d '{ +> "jsonrpc": "2.0", +> "method": "session.login_with_password", +> "id": 0 +> }' +HTTP/1.1 500 Internal Error +content-length: 308 +content-type:text/html +connection:close +cache-control:no-cache, no-store + +

HTTP 500 internal server error

An unexpected error occurred; + please wait a while and try again. If the problem persists, please contact your + support representative.

Additional information

Jsonrpc.Malformed_metho +d_request("{jsonrpc=...,method=...,id=...}") +``` + +All other failures are reported with a more structured error response, to +allow better automatic response to failures, proper internationalisation of +any error message, and easier debugging. + +On the wire, these are transmitted like this when using the XML-RPC protocol: + +```xml + + + Status + Failure + + + ErrorDescription + + + + MAP_DUPLICATE_KEY + Customer + eSpiel Inc. + eSpiel Incorporated + + + + + +``` + +Note that `ErrorDescription` value is an array of string values. The +first element of the array is an error code; the remainder of the array are +strings representing error parameters relating to that code. In this case, +the client has attempted to add the mapping _Customer -> +eSpiel Incorporated_ to a Map, but it already contains the mapping +_Customer -> eSpiel Inc._, hence the request has failed. + +When using the JSON-RPC protocol v2.0, the above error is transmitted as: + +```json +{ + "jsonrpc": "2.0", + "error": { + "code": 1, + "message": "MAP_DUPLICATE_KEY", + "data": [ + "Customer", + "eSpiel Inc.", + "eSpiel Incorporated" + ] + }, + "id": 3 +} +``` + +Finally, when using the JSON-RPC protocol v1.0: + +```json +{ + "result": null, + "error": [ + "MAP_DUPLICATE_KEY", + "Customer", + "eSpiel Inc.", + "eSpiel Incorporated" + ], + "id": "xyz" +} +``` + +Each possible error code is documented in the following section. + +## Error Codes +{{#errors}} + +### {{{error_code}}} + +{{{error_description}}} + +_Signature:_ + +``` +{{{error_code_unescaped}}}({{parameters}}) +``` +{{/errors}} \ No newline at end of file diff --git a/ocaml/idl/templates/class.mustache b/ocaml/idl/templates/class.mustache new file mode 100644 index 00000000000..65c7c294d22 --- /dev/null +++ b/ocaml/idl/templates/class.mustache @@ -0,0 +1,89 @@ +--- + layout: doc +--- + +# Class: {{{class_name}}} +{{#class_deprecated}} + +**This class is deprecated.** +{{/class_deprecated}} +{{#class_removed}} + +**This class is removed.** +{{/class_removed}} +{{#has_descr}} + +{{{class_descr}}} +{{/has_descr}} + +## Fields for class: {{{class_name}}} +{{#has_fields}} + +|Field |Type |Qualifier |Description | +|:-------------------|:-------------------|:--------------|:---------------------------------------| +{{/has_fields}} +{{#fields}} +|{{{field_name}}}|{{{field_type}}}|{{{field_ctor}}}|{{#field_deprecated}}**Deprecated.** {{/field_deprecated}}{{#field_removed}}**Removed.** {{/field_removed}}{{{field_descr}}}| +{{/fields}} +{{#is_event}} +|snapshot |object record |_RO/runtime_ |The record of the database object that was added, changed or deleted| +{{/is_event}} +{{^has_fields}} + +Class {{{class_name}}} has no fields. +{{/has_fields}} + +## RPCs associated with class: {{{class_name}}} +{{#all_rpcs}} + +### RPC name: {{{rpc_name_escaped}}} +{{#rpc_deprecated}} + +**This message is deprecated.** +{{/rpc_deprecated}} +{{#rpc_removed}} + +**This message is removed.** +{{/rpc_removed}} + +_Overview:_ +{{#rpc_has_descr}} + +{{{rpc_descr}}} +{{/rpc_has_descr}} + +_Signature:_ + +``` +{{{return_type}}} {{{rpc_name}}} ({{#session}}session ref session_ref{{#has_rpc_params}}, {{/has_rpc_params}}{{/session}}{{{rpc_param_csv}}}) +``` + +_Arguments:_ + +|type |name |description | +|:-----------------------------|:-----------------------------|:---------------------------------------| +{{#session}} +|session ref |session_ref |Reference to a valid session | +{{/session}} +{{#rpc_params}} +|{{{param_type}}}|{{{param_name}}}|{{{param_descr}}}| +{{/rpc_params}} + +{{#has_rbac}} +_Minimum Role:_ {{min_role}} + +{{/has_rbac}} +_Return Type:_ `{{{return_type}}}` +{{^returns_void}} + +{{{return_descr}}} +{{/returns_void}} +{{#has_error_codes}} + +_Possible Error Codes:_ {{{error_codes_csv}}} +{{/has_error_codes}} +{{/all_rpcs}} +{{^has_rpcs}} + +Class {{{class_name}}} has no RPCs associated with it. +{{/has_rpcs}} \ No newline at end of file diff --git a/ocaml/idl/templates/classes.mustache b/ocaml/idl/templates/classes.mustache new file mode 100644 index 00000000000..3730d77d0bc --- /dev/null +++ b/ocaml/idl/templates/classes.mustache @@ -0,0 +1,13 @@ +--- + layout: doc +--- + +# Classes + +The following classes are defined: + +|Name |Description | +|:-------------------|:---------------------------------------------------------------------| +{{#classes}} +|[{{{name}}}](@root@management-api/class-{{{name_lower}}}.html)|{{{description}}}| +{{/classes}} diff --git a/ocaml/idl/templates/relationships.mustache b/ocaml/idl/templates/relationships.mustache new file mode 100644 index 00000000000..a8567f5c6be --- /dev/null +++ b/ocaml/idl/templates/relationships.mustache @@ -0,0 +1,19 @@ +--- + layout: doc +--- + +# Relationships Between Classes + +Fields that are bound together are shown in the following table: + +|_object.field_ |_object.field_ |_relationship_ | +|:---------------------------------------|:---------------------------------------|:--------------| +{{#relationships}} +|{{{a_field}}}|{{{b_field}}}|{{relationship}}| +{{/relationships}} + +The following figure represents bound fields (as specified above) +diagramatically, using crow's foot notation to specify one-to-one, +one-to-many, or many-to-many relationships: + +![Class relationships](classes.png 'Class relationships') \ No newline at end of file diff --git a/ocaml/idl/templates/toc.mustache b/ocaml/idl/templates/toc.mustache new file mode 100644 index 00000000000..01f81f0982f --- /dev/null +++ b/ocaml/idl/templates/toc.mustache @@ -0,0 +1,15 @@ +- title: API Reference + url: @root@api-ref-autogen.html + subfolderlist: + - title: Classes + url: @root@management-api/classes.html +{{#classes}} + - title: Class:{{{name}}} + url: @root@management-api/class-{{{name_lower}}}.html +{{/classes}} + - title: Relationships Between Classes + url: @root@management-api/relationships-between-classes.html + - title: Types + url: @root@management-api/types.html + - title: Error Handling + url: @root@management-api/api-ref-autogen-errors.html diff --git a/ocaml/idl/templates/types.mustache b/ocaml/idl/templates/types.mustache new file mode 100644 index 00000000000..197a8ca5a0e --- /dev/null +++ b/ocaml/idl/templates/types.mustache @@ -0,0 +1,40 @@ +--- + layout: doc +--- + +# Types + +## Primitives + +The following primitive types are used to specify methods and fields in +the API Reference: + +|Type |Description | +|:-------|:-------------------------------------------| +|string |text strings | +|int |64-bit integers | +|float |IEEE double-precision floating-point numbers| +|bool |boolean | +|datetime|date and timestamp | + +## Higher-order types + +The following type constructors are used: + +|Type |Description | +|:-----------------|:-------------------------------------------------------| +|_c_ ref |reference to an object of class _c_ | +|_t_ set |a set of elements of type _t_ | +|(_a -> b_) map |a table mapping values of type _a_ to values of type _b_| + +## Enumeration types + +The following enumeration types are used: +{{#enums}} + +|enum {{{enum}}}| | +|:---------------------------------------|:---------------------------------------| +{{#enum_options}} +|{{{option_name}}}|{{{option_descr}}}| +{{/enum_options}} +{{/enums}} \ No newline at end of file From ae7480c84960f4ce3693d27858db8eeb58d5cb95 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Sun, 17 Dec 2023 19:24:26 +0000 Subject: [PATCH 033/149] Generate the C enums with templates. Signed-off-by: Konstantina Chremmou --- ocaml/sdk-gen/c/gen_c_binding.ml | 218 ++++-------------- ocaml/sdk-gen/c/templates/xen_enum.c.mustache | 98 ++++++++ ocaml/sdk-gen/c/templates/xen_enum.h.mustache | 91 ++++++++ .../c/templates/xen_enum_internal.h.mustache | 51 ++++ .../powershell/gen_powershell_binding.ml | 8 +- 5 files changed, 285 insertions(+), 181 deletions(-) create mode 100644 ocaml/sdk-gen/c/templates/xen_enum.c.mustache create mode 100644 ocaml/sdk-gen/c/templates/xen_enum.h.mustache create mode 100644 ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache diff --git a/ocaml/sdk-gen/c/gen_c_binding.ml b/ocaml/sdk-gen/c/gen_c_binding.ml index 5203b86f2e0..e9035a88c7f 100644 --- a/ocaml/sdk-gen/c/gen_c_binding.ml +++ b/ocaml/sdk-gen/c/gen_c_binding.ml @@ -83,11 +83,7 @@ let rec main () = all_headers := List.map (fun x -> x.name) filtered_classes ; - TypeSet.iter (gen_enum write_enum_decl decl_filename include_dir) !enums ; - TypeSet.iter (gen_enum write_enum_impl impl_filename src_dir) !enums ; - TypeSet.iter - (gen_enum write_enum_internal_decl internal_decl_filename include_dir) - !enums ; + TypeSet.iter render_enum !enums ; maps := TypeSet.add (Map (String, Int)) !maps ; maps := TypeSet.add (Map (Int, Int)) !maps ; @@ -140,17 +136,6 @@ and gen_class f g clas targetdir = let out_chan = open_out (Filename.concat targetdir (g clas.name)) in Fun.protect (fun () -> f clas out_chan) ~finally:(fun () -> close_out out_chan) -and gen_enum f g targetdir = function - | Enum (name, _) as x -> - if not (List.mem name !all_headers) then - all_headers := name :: !all_headers ; - let out_chan = open_out (Filename.concat targetdir (g name)) in - Fun.protect - (fun () -> f x out_chan) - ~finally:(fun () -> close_out out_chan) - | _ -> - assert false - and gen_map f g targetdir = function | Map (l, r) -> let name = mapname l r in @@ -669,174 +654,49 @@ and hash_include n = else sprintf "#include <%s>" (decl_filename n) -and write_enum_decl x out_chan = - match x with - | Enum (name, contents) -> - let print format = fprintf out_chan format in - let protect = protector name in - let tn = typename name in - - print_h_header out_chan protect ; +and replace_dashes x = + Astring.String.map (fun y -> match y with '-' -> '_' | _ -> y) x - print - "\n\ - %s\n\n\n\ - enum %s\n\ - {\n\ - %s\n\ - };\n\n\n\ - typedef struct %s_set\n\ - {\n\ - \ size_t size;\n\ - \ enum %s contents[];\n\ - } %s_set;\n\n\ - %s\n\ - extern %s_set *\n\ - %s_set_alloc(size_t size);\n\n\ - %s\n\n\n\ - %s\n\ - extern const char *\n\ - %s_to_string(enum %s val);\n\n\n\ - %s\n\ - extern enum %s\n\ - %s_from_string(xen_session *session, const char *str);\n\n" - (hash_include "common") tn - (joined ",\n\n" (enum_entry name) - (contents - @ [("undefined", "Unknown to this version of the bindings.")] - ) - ) - tn tn tn - (Helper.comment true (sprintf "Allocate a %s_set of the given size." tn)) - tn tn - (decl_free (sprintf "%s_set" tn) "*set" false "set") - (Helper.comment true - "Return the name corresponding to the given code. This string must \ - not be modified or freed." - ) - tn tn - (Helper.comment true - "Return the correct code for the given string, or set the session \ - object to failure and return an undefined value if the given \ - string does not match a known code." - ) - tn tn ; - - print_h_footer out_chan - | _ -> - () - -and enum_entry enum_name = function - | n, c -> - sprintf "%s\n XEN_%s_%s" - (Helper.comment true ~indent:4 c) - (String.uppercase_ascii enum_name) - (Astring.String.map - (fun x -> match x with '-' -> '_' | _ -> x) - (String.uppercase_ascii n) - ) - -and write_enum_impl x out_chan = +and render_enum x = match x with | Enum (name, contents) -> - let print format = fprintf out_chan format in - let tn = typename name in - - print - "%s\n\n\ - #include \n\n\ - %s\n\ - %s\n\ - %s\n\n\n\ - /*\n\ - \ * Maintain this in the same order as the enum declaration!\n\ - \ */\n\ - static const char *lookup_table[] =\n\ - {\n\ - %s\n\ - };\n\n\n\ - extern %s_set *\n\ - %s_set_alloc(size_t size)\n\ - {\n\ - \ return calloc(1, sizeof(%s_set) +\n\ - \ size * sizeof(enum %s));\n\ - }\n\n\n\ - extern void\n\ - %s_set_free(%s_set *set)\n\ - {\n\ - \ free(set);\n\ - }\n\n\n\ - const char *\n\ - %s_to_string(enum %s val)\n\ - {\n\ - \ return lookup_table[val];\n\ - }\n\n\n\ - extern enum %s\n\ - %s_from_string(xen_session *session, const char *str)\n\ - {\n\ - \ (void)session;\n\ - \ return ENUM_LOOKUP(str, lookup_table);\n\ - }\n\n\n\ - const abstract_type %s_abstract_type_ =\n\ - \ {\n\ - \ .XEN_API_TYPE = ENUM,\n\ - \ .enum_marshaller =\n\ - \ (const char *(*)(int))&%s_to_string,\n\ - \ .enum_demarshaller =\n\ - \ (int (*)(xen_session *, const char *))&%s_from_string\n\ - \ };\n\n\n" - Licence.bsd_two_clause (hash_include "internal") (hash_include name) - (hash_include (name ^ "_internal")) - (enum_lookup_entries (contents @ [("undefined", "")])) - tn tn tn tn tn tn tn tn tn tn tn tn tn ; - - if name <> "event_operation" then - print - "const abstract_type %s_set_abstract_type_ =\n\ - \ {\n\ - \ .XEN_API_TYPE = SET,\n\ - \ .child = &%s_abstract_type_\n\ - \ };\n\n\n" - tn tn - | _ -> - () - -and enum_lookup_entries contents = joined ",\n" enum_lookup_entry contents - -and enum_lookup_entry = function n, _ -> sprintf " \"%s\"" n - -and write_enum_internal_decl x out_chan = - match x with - | Enum (name, _) -> - let print format = fprintf out_chan format in - let protect = protector (sprintf "%s_internal" name) in - let tn = typename name in - - let set_abstract_type = - if name = "event_operations" then - "" - else - sprintf "extern const abstract_type %s_set_abstract_type_;\n" tn + if not (List.mem name !all_headers) then + all_headers := name :: !all_headers ; + let json = + `O + [ + ("enum_name", `String name) + ; ("enum_name_upper", `String (String.uppercase_ascii name)) + ; ("event_operations", `Bool (name = "event_operation")) + ; ( "enum_values" + , `A + (List.map + (fun (n, c) -> + `O + [ + ("enum_value", `String n) + ; ("enum_value_doc", `String c) + ; ( "enum_value_upper" + , `String (replace_dashes (String.uppercase_ascii n)) + ) + ] + ) + contents + ) + ) + ] in - - print - "%s\n\n\n\ - %s\n\n\n\ - #ifndef %s\n\ - #define %s\n\n\n\ - %s\n\n\n\ - extern const abstract_type %s_abstract_type_;\n\ - %s\n\n\ - #endif\n" - Licence.bsd_two_clause - (Helper.comment false - (sprintf - "Declarations of the abstract types used during demarshalling of \ - enum %s. Internal to this library -- do not use from outside." - tn - ) + render_file + ( "xen_enum_internal.h.mustache" + , sprintf "include/xen_%s_internal.h" name ) - protect protect (hash_include "internal") tn set_abstract_type + json templates_dir destdir ; + render_file + ("xen_enum.h.mustache", sprintf "include/xen/api/xen_%s.h" name) + json templates_dir destdir ; + render_file + ("xen_enum.c.mustache", sprintf "src/xen_%s.c" name) + json templates_dir destdir | _ -> () diff --git a/ocaml/sdk-gen/c/templates/xen_enum.c.mustache b/ocaml/sdk-gen/c/templates/xen_enum.c.mustache new file mode 100644 index 00000000000..421c9015a6f --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_enum.c.mustache @@ -0,0 +1,98 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include + +#include "xen_internal.h" +#include +#include "xen_{{{enum_name}}}_internal.h" + + +/* + * Maintain this in the same order as the enum declaration! + */ +static const char *lookup_table[] = +{ +{{#enum_values}} + "{{{enum_value}}}", +{{/enum_values}} + "undefined" +}; + + +extern xen_{{{enum_name}}}_set * +xen_{{{enum_name}}}_set_alloc(size_t size) +{ + return calloc(1, sizeof(xen_{{{enum_name}}}_set) + + size * sizeof(enum xen_{{{enum_name}}})); +} + + +extern void +xen_{{{enum_name}}}_set_free(xen_{{{enum_name}}}_set *set) +{ + free(set); +} + + +const char * +xen_{{{enum_name}}}_to_string(enum xen_{{{enum_name}}} val) +{ + return lookup_table[val]; +} + + +extern enum xen_{{{enum_name}}} +xen_{{{enum_name}}}_from_string(xen_session *session, const char *str) +{ + (void)session; + return ENUM_LOOKUP(str, lookup_table); +} + + +const abstract_type xen_{{{enum_name}}}_abstract_type_ = + { + .XEN_API_TYPE = ENUM, + .enum_marshaller = + (const char *(*)(int))&xen_{{{enum_name}}}_to_string, + .enum_demarshaller = + (int (*)(xen_session *, const char *))&xen_{{{enum_name}}}_from_string + }; + + +{{^event_operations}} +const abstract_type xen_{{{enum_name}}}_set_abstract_type_ = + { + .XEN_API_TYPE = SET, + .child = &xen_{{{enum_name}}}_abstract_type_ + }; + + +{{/event_operations}} diff --git a/ocaml/sdk-gen/c/templates/xen_enum.h.mustache b/ocaml/sdk-gen/c/templates/xen_enum.h.mustache new file mode 100644 index 00000000000..824179cf2d3 --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_enum.h.mustache @@ -0,0 +1,91 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef XEN_{{{enum_name_upper}}}_H +#define XEN_{{{enum_name_upper}}}_H + + +#include + + +enum xen_{{{enum_name}}} +{ +{{#enum_values}} + /** + * {{{enum_value_doc}}} + */ + XEN_{{{enum_name_upper}}}_{{{enum_value_upper}}}, + +{{/enum_values}} + /** + * Unknown to this version of the bindings. + */ + XEN_{{{enum_name_upper}}}_UNDEFINED +}; + + +typedef struct xen_{{{enum_name}}}_set +{ + size_t size; + enum xen_{{{enum_name}}} contents[]; +} xen_{{{enum_name}}}_set; + +/** + * Allocate a xen_{{{enum_name}}}_set of the given size. + */ +extern xen_{{{enum_name}}}_set * +xen_{{{enum_name}}}_set_alloc(size_t size); + +/** + * Free the given xen_{{{enum_name}}}_set. The given set must have been + * allocated by this library. + */ +extern void +xen_{{{enum_name}}}_set_free(xen_{{{enum_name}}}_set *set); + + +/** + * Return the name corresponding to the given code. This string must + * not be modified or freed. + */ +extern const char * +xen_{{{enum_name}}}_to_string(enum xen_{{{enum_name}}} val); + + +/** + * Return the correct code for the given string, or set the session + * object to failure and return an undefined value if the given string does + * not match a known code. + */ +extern enum xen_{{{enum_name}}} +xen_{{{enum_name}}}_from_string(xen_session *session, const char *str); + + +#endif diff --git a/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache b/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache new file mode 100644 index 00000000000..b9731686edb --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache @@ -0,0 +1,51 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + + +/* + * Declarations of the abstract types used during demarshalling of enum + * xen_{{{enum_name}}}. Internal to this library -- do not use from outside. + */ + + +#ifndef XEN_{{{enum_name_upper}}}_INTERNAL_H +#define XEN_{{{enum_name_upper}}}_INTERNAL_H + + +#include "xen_internal.h" + + +extern const abstract_type xen_{{{enum_name}}}_abstract_type_; +{{^event_operations}} +extern const abstract_type xen_{{{enum_name}}}_set_abstract_type_; +{{/event_operations}} + + +#endif diff --git a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml index 3f49c7deff8..b455d010486 100644 --- a/ocaml/sdk-gen/powershell/gen_powershell_binding.ml +++ b/ocaml/sdk-gen/powershell/gen_powershell_binding.ml @@ -18,6 +18,7 @@ module TypeSet = Set.Make (struct end) let destdir = "autogen/src" + let templdir = "templates" type cmdlet = {filename: string; content: string} @@ -111,9 +112,12 @@ and gen_http_action action = let stem = get_http_action_stem name in let arg_name = function | String_query_arg x | Int64_query_arg x -> - pascal_case_rec x + pascal_case_rec x | Bool_query_arg x -> - if String.lowercase_ascii x = "host" then "IsHost" else pascal_case_rec x + if String.lowercase_ascii x = "host" then + "IsHost" + else + pascal_case_rec x | Varargs_query_arg -> "Args" in From 37821d8828a8366c79676027cc73ffac440e01c8 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Fri, 8 Mar 2024 21:46:22 +0000 Subject: [PATCH 034/149] Corrections as per code review. Signed-off-by: Konstantina Chremmou --- ocaml/doc/basics.md | 4 +-- ocaml/doc/wire-protocol.md | 10 ++++---- ocaml/idl/datamodel.ml | 5 +++- ocaml/idl/datamodel_certificate.ml | 3 ++- ocaml/idl/markdown_backend.ml | 10 +++++--- ocaml/idl/templates/api_errors.mustache | 19 ++++++++++---- ocaml/idl/templates/class.mustache | 4 +-- ocaml/idl/templates/relationships.mustache | 9 +++---- ocaml/idl/templates/types.mustache | 30 ++++++++++++---------- 9 files changed, 56 insertions(+), 38 deletions(-) diff --git a/ocaml/doc/basics.md b/ocaml/doc/basics.md index 9a75a16087b..b1812da023d 100644 --- a/ocaml/doc/basics.md +++ b/ocaml/doc/basics.md @@ -5,12 +5,12 @@ # API Basics This document defines the XenServer Management API - an interface for remotely -configuring and controlling virtualised guests running on a Xen-enabled host. +configuring and controlling virtualized guests running on a Xen-enabled host. The API is presented here as a set of Remote Procedure Calls (RPCs). There are two supported wire formats, one based upon [XML-RPC](http://xmlrpc.scripting.com/spec.html) and one based upon [JSON-RPC](http://www.jsonrpc.org) (v1.0 and v2.0 are both -recognised). No specific language bindings are prescribed, although examples +recognized). No specific language bindings are prescribed, although examples will be given in the python programming language. Although we adopt some terminology from object-oriented programming, diff --git a/ocaml/doc/wire-protocol.md b/ocaml/doc/wire-protocol.md index e42a0f2da2c..808d021154d 100644 --- a/ocaml/doc/wire-protocol.md +++ b/ocaml/doc/wire-protocol.md @@ -182,7 +182,7 @@ following manner: * our `void` type is transmitted as an empty string. -Both versions 1.0 and 2.0 of the JSON-RPC wire format are recognised and, +Both versions 1.0 and 2.0 of the JSON-RPC wire format are recognized and, depending on your client library, you can use either of them. ### JSON-RPC v1.0 @@ -490,7 +490,7 @@ session reference is returned under the key `Value` in the resulting dictionary ... "version", "originator")['Value'] ``` -This is what the call looks like when serialised +This is what the call looks like when serialized ```xml @@ -534,7 +534,7 @@ Once a reference to a VM has been acquired, a lifecycle operation may be invoked In this case the `start` message has been rejected, because the VM is a template, and so an error response has been returned. These high-level errors are returned as structured data (rather than as XML-RPC faults), -allowing them to be internationalised. +allowing them to be internationalized. Rather than querying fields individually, whole _records_ may be returned at once. To retrieve the record of a single object as a python dictionary: @@ -579,7 +579,7 @@ reference: ... "user", "passwd", "version", "originator") ``` -`pyjsonrpc` uses the JSON-RPC protocol v2.0, so this is what the serialised +`pyjsonrpc` uses the JSON-RPC protocol v2.0, so this is what the serialized request looks like: ```json @@ -627,7 +627,7 @@ Once a reference to a VM has been acquired, a lifecycle operation may be invoked In this case the `start` message has been rejected because the VM is a template, hence an error response has been returned. These high-level -errors are returned as structured data, allowing them to be internationalised. +errors are returned as structured data, allowing them to be internationalized. Rather than querying fields individually, whole _records_ may be returned at once. To retrieve the record of a single object as a python dictionary: diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index d7bff0266a9..8f452b47069 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -2057,7 +2057,10 @@ module Bond = struct let t = create_obj ~in_db:true ~in_product_since:rel_miami ~in_oss_since:None ~persist:PersistEverything ~gen_constructor_destructor:false ~name:_bond - ~descr:"" ~gen_events:true ~doccomments:[] + ~descr: + "A Network bond that combines physical network interfaces, also known \ + as link aggregation" + ~gen_events:true ~doccomments:[] ~messages_default_allowed_roles:_R_POOL_OP ~doc_tags:[Networking] ~messages:[create; destroy; set_mode; set_property] ~contents: diff --git a/ocaml/idl/datamodel_certificate.ml b/ocaml/idl/datamodel_certificate.ml index b18a20e7656..ac77887b9f0 100644 --- a/ocaml/idl/datamodel_certificate.ml +++ b/ocaml/idl/datamodel_certificate.ml @@ -34,7 +34,8 @@ let certificate_type = ) let t = - create_obj ~name:_certificate ~descr:"Description" ~doccomments:[] + create_obj ~name:_certificate + ~descr:"An X509 certificate used for TLS connections" ~doccomments:[] ~gen_constructor_destructor:false ~gen_events:true ~in_db:true ~lifecycle ~persist:PersistEverything ~in_oss_since:None ~messages_default_allowed_roles:_R_READ_ONLY diff --git a/ocaml/idl/markdown_backend.ml b/ocaml/idl/markdown_backend.ml index c4bbd538fe1..38ec1c8caef 100644 --- a/ocaml/idl/markdown_backend.ml +++ b/ocaml/idl/markdown_backend.ml @@ -185,7 +185,10 @@ let generate_class cls = ( "field_name" , `String (pad_right - (escape (Datamodel_utils.wire_name_of_field field)) + ("`" + ^ Datamodel_utils.wire_name_of_field field + ^ "`" + ) col_width_20 ) ) @@ -298,7 +301,8 @@ let generate_class cls = [ ( "param_name" , `String - (pad_right (escape p.param_name) + (pad_right + ("`" ^ p.param_name ^ "`") col_width_30 ) ) @@ -357,7 +361,7 @@ let generate_types system = | Enum (name, options) -> `O [ - ("enum", `String (pad_right name (col_width_40 - 5))) + ("enum", `String name) ; ( "enum_options" , `A (options diff --git a/ocaml/idl/templates/api_errors.mustache b/ocaml/idl/templates/api_errors.mustache index e1a4b95fbaa..d2e877e25d1 100644 --- a/ocaml/idl/templates/api_errors.mustache +++ b/ocaml/idl/templates/api_errors.mustache @@ -9,15 +9,19 @@ or RPC level, the server may send an HTTP 500 error response, or the client may simulate the same. The client must be prepared to handle these errors, though they may be treated as fatal. -On the wire, these are transmitted in a form similar to this when using the -XML-RPC protocol: +For example, the following malformed request when using the XML-RPC protocol: -``` +```sh $curl -D - -X POST https://server -H 'Content-Type: application/xml' \ > -d ' > > session.logout > ' +``` + +results to the following response: + +```sh HTTP/1.1 500 Internal Error content-length: 297 content-type:text/html @@ -32,13 +36,18 @@ t;close_tag", "open_tag", _) When using the JSON-RPC protocol: -``` +```sh $curl -D - -X POST https://server/jsonrpc -H 'Content-Type: application/json' \ > -d '{ > "jsonrpc": "2.0", > "method": "session.login_with_password", > "id": 0 > }' +``` + +the response is: + +```sh HTTP/1.1 500 Internal Error content-length: 308 content-type:text/html @@ -52,7 +61,7 @@ d_request("{jsonrpc=...,method=...,id=...}") ``` All other failures are reported with a more structured error response, to -allow better automatic response to failures, proper internationalisation of +allow better automatic response to failures, proper internationalization of any error message, and easier debugging. On the wire, these are transmitted like this when using the XML-RPC protocol: diff --git a/ocaml/idl/templates/class.mustache b/ocaml/idl/templates/class.mustache index 65c7c294d22..1112c851642 100644 --- a/ocaml/idl/templates/class.mustache +++ b/ocaml/idl/templates/class.mustache @@ -60,10 +60,10 @@ _Signature:_ _Arguments:_ -|type |name |description | +|Type |Name |Description | |:-----------------------------|:-----------------------------|:---------------------------------------| {{#session}} -|session ref |session_ref |Reference to a valid session | +|`session ref` |`session_ref` |Reference to a valid session | {{/session}} {{#rpc_params}} |{{{param_type}}}|{{{param_name}}}|{{{param_descr}}}| diff --git a/ocaml/idl/templates/relationships.mustache b/ocaml/idl/templates/relationships.mustache index a8567f5c6be..6678c875681 100644 --- a/ocaml/idl/templates/relationships.mustache +++ b/ocaml/idl/templates/relationships.mustache @@ -6,14 +6,13 @@ Fields that are bound together are shown in the following table: -|_object.field_ |_object.field_ |_relationship_ | -|:---------------------------------------|:---------------------------------------|:--------------| +|_object.field_ |_object.field_ |_relationship_ | +|:-------------------------------------|:-------------------------------------|:--------------| {{#relationships}} |{{{a_field}}}|{{{b_field}}}|{{relationship}}| {{/relationships}} -The following figure represents bound fields (as specified above) -diagramatically, using crow's foot notation to specify one-to-one, -one-to-many, or many-to-many relationships: +The following figure represents bound fields (as specified above) using crow's +foot notation to specify one-to-one, one-to-many, or many-to-many relationships: ![Class relationships](classes.png 'Class relationships') \ No newline at end of file diff --git a/ocaml/idl/templates/types.mustache b/ocaml/idl/templates/types.mustache index 197a8ca5a0e..f1ebc136899 100644 --- a/ocaml/idl/templates/types.mustache +++ b/ocaml/idl/templates/types.mustache @@ -9,31 +9,33 @@ The following primitive types are used to specify methods and fields in the API Reference: -|Type |Description | -|:-------|:-------------------------------------------| -|string |text strings | -|int |64-bit integers | -|float |IEEE double-precision floating-point numbers| -|bool |boolean | -|datetime|date and timestamp | +|Type |Description | +|:---------|:-------------------------------------------| +|`string` |text strings | +|`int` |64-bit integers | +|`float` |IEEE double-precision floating-point numbers| +|`bool` |boolean | +|`datetime`|date and timestamp | ## Higher-order types The following type constructors are used: -|Type |Description | -|:-----------------|:-------------------------------------------------------| -|_c_ ref |reference to an object of class _c_ | -|_t_ set |a set of elements of type _t_ | -|(_a -> b_) map |a table mapping values of type _a_ to values of type _b_| +|Type |Description | +|:-------------|:-------------------------------------------------------| +|`c ref` |reference to an object of class `c` | +|`t set` |a set of elements of type `t` | +|`(a -> b) map`|a table mapping values of type `a` to values of type `b`| ## Enumeration types The following enumeration types are used: {{#enums}} -|enum {{{enum}}}| | -|:---------------------------------------|:---------------------------------------| +### Enum {{{enum}}} + +|Named value |Description | +|:---------------------------------------|:----------------------------------------------------| {{#enum_options}} |{{{option_name}}}|{{{option_descr}}}| {{/enum_options}} From 311b79b9fb079e34b25d9246520489d360985ee0 Mon Sep 17 00:00:00 2001 From: Bernhard Kaindl Date: Mon, 11 Mar 2024 06:38:28 +0100 Subject: [PATCH 035/149] Github Actions: Update to falti/dotenv-action@v1 to fix Node 16 warnings Co-authored-by: Pau Ruiz Safont Signed-off-by: Bernhard Kaindl --- .github/workflows/1.249-lcm.yml | 2 +- .github/workflows/format.yml | 2 +- .github/workflows/setup-xapi-environment/action.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/1.249-lcm.yml b/.github/workflows/1.249-lcm.yml index 5fec8bef8d8..e1e78b7eb4d 100644 --- a/.github/workflows/1.249-lcm.yml +++ b/.github/workflows/1.249-lcm.yml @@ -40,7 +40,7 @@ jobs: - name: Load environment file id: dotenv - uses: falti/dotenv-action@v1.0.2 + uses: falti/dotenv-action@v1 - name: Retrieve date for cache key (year-week) id: cache-key diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index b15173805cf..b7ea7288627 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -22,7 +22,7 @@ jobs: - name: Load environment file id: dotenv - uses: falti/dotenv-action@v1.0.4 + uses: falti/dotenv-action@v1 - name: Update Ubuntu repositories run: sudo apt-get update diff --git a/.github/workflows/setup-xapi-environment/action.yml b/.github/workflows/setup-xapi-environment/action.yml index 702162d42f3..e25e0e184fb 100644 --- a/.github/workflows/setup-xapi-environment/action.yml +++ b/.github/workflows/setup-xapi-environment/action.yml @@ -25,7 +25,7 @@ runs: - name: Load environment file id: dotenv - uses: falti/dotenv-action@v1.0.4 + uses: falti/dotenv-action@v1 - name: Update Ubuntu repositories shell: bash From 51521bf11de3617b6e43557a9a4d9fe908920fcc Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 11 Mar 2024 16:52:54 +0000 Subject: [PATCH 036/149] CA-389496: Avoid configuration conflicts for rotating xapi logs The custom logrotation setup for xapi sometimes conflicted with the normal one. Remove the former. Install the xapi logrotate configuration has been moved to be managed as part of the normal logrotation, with the global settings removed. Duplicate settings in the configuration have been removed. Signed-off-by: Pau Ruiz Safont --- scripts/Makefile | 4 +--- scripts/xapi-logrotate.conf | 20 -------------------- scripts/xapi-logrotate.cron | 3 --- scripts/xapi-logrotate.sh | 13 ------------- 4 files changed, 1 insertion(+), 39 deletions(-) delete mode 100644 scripts/xapi-logrotate.cron delete mode 100755 scripts/xapi-logrotate.sh diff --git a/scripts/Makefile b/scripts/Makefile index 17c8e383fac..d2ab8f5d381 100644 --- a/scripts/Makefile +++ b/scripts/Makefile @@ -45,8 +45,7 @@ install: $(IPROG) xapi-health-check $(DESTDIR)$(LIBEXECDIR) $(IPROG) mail-alarm $(DESTDIR)$(LIBEXECDIR) $(IDATA) audit-logrotate $(DESTDIR)/etc/logrotate.d/audit - $(IDATA) xapi-logrotate.conf $(DESTDIR)$(ETCXENDIR) - $(IPROG) xapi-logrotate.sh $(DESTDIR)$(LIBEXECDIR) + $(IDATA) xapi-logrotate.conf $(DESTDIR)/etc/logrotate.d/xapi $(IPROG) xapi-tracing-log-trim.sh $(DESTDIR)$(LIBEXECDIR) $(IPROG) xapi-wait-init-complete $(DESTDIR)$(OPTDIR)/bin $(IPROG) xapi-autostart-vms $(DESTDIR)$(OPTDIR)/bin @@ -158,7 +157,6 @@ install: $(IPROG) certificate-check $(DESTDIR)/etc/cron.daily/ $(IPROG) certificate-refresh $(DESTDIR)/etc/cron.hourly/ mkdir -p $(DESTDIR)/etc/cron.d - $(IDATA) xapi-logrotate.cron $(DESTDIR)/etc/cron.d/xapi-logrotate.cron $(IDATA) xapi-tracing-log-trim.cron $(DESTDIR)/etc/cron.d/xapi-tracing-log-trim.cron mkdir -p $(DESTDIR)/opt/xensource/gpg # templates diff --git a/scripts/xapi-logrotate.conf b/scripts/xapi-logrotate.conf index b50677e72f2..e9d4845d582 100644 --- a/scripts/xapi-logrotate.conf +++ b/scripts/xapi-logrotate.conf @@ -1,20 +1,3 @@ -# see "man logrotate" for details -# rotate log files daily -daily - -# keep one months worth of backlogs -rotate 31 - -# create new (empty) log files after rotating old ones -create - -# compress log files -compress -delaycompress - -# All the general settings above are copied from /etc/logrotate.conf -# as installed by the logrotate RPM. - /var/log/xensource.log { missingok @@ -26,9 +9,6 @@ delaycompress # When rotating, remove any rotated logs older than this many days. maxage 31 - # Rotate when file exceeds this size, even if we rotated it today already. - maxsize 100M - # Keep up to this many old files. # (When considering total size, expect a compression factor around 10-20.) rotate 100 diff --git a/scripts/xapi-logrotate.cron b/scripts/xapi-logrotate.cron deleted file mode 100644 index 3f4506fb1f6..00000000000 --- a/scripts/xapi-logrotate.cron +++ /dev/null @@ -1,3 +0,0 @@ -# Run at 40 minutes past the hour, -# every hour, day-of-month, month, day-of-week -40 * * * * root /opt/xensource/libexec/xapi-logrotate.sh diff --git a/scripts/xapi-logrotate.sh b/scripts/xapi-logrotate.sh deleted file mode 100755 index b4a5935feae..00000000000 --- a/scripts/xapi-logrotate.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -# If we're using the old partitioning scheme there's no need to run this. -if ! grep -q /var/log /proc/mounts; then - exit 0 -fi - -/usr/sbin/logrotate /etc/xensource/xapi-logrotate.conf -EXITVALUE=$? -if [ $EXITVALUE != 0 ]; then - /usr/bin/logger -t $0 "ALERT exited abnormally with [$EXITVALUE]" -fi -exit 0 From ef9f18ba829cd14b28290409aa8b0549d4866bd7 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Wed, 13 Mar 2024 08:39:30 +0000 Subject: [PATCH 037/149] Bump maven plugin versions Also update URL in `pom.xml` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/java/autogen/xen-api/pom.xml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml index abaed44d833..66e1b633db2 100644 --- a/ocaml/sdk-gen/java/autogen/xen-api/pom.xml +++ b/ocaml/sdk-gen/java/autogen/xen-api/pom.xml @@ -1,6 +1,6 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 com.xenserver xen-api @@ -8,16 +8,16 @@ jar XenServer Java SDK Mavenized build of the XenServer SDK for Java. - https://www.citrix.com/community/citrix-developer/citrix-hypervisor-developer + https://docs.xenserver.com/en-us/xenserver/8/developer Cloud Software Group, Inc. https://www.cloud.com - BSD 2-Clause License - http://opensource.org/licenses/BSD-2-Clause - repo + BSD 2-Clause License + http://opensource.org/licenses/BSD-2-Clause + repo @@ -81,7 +81,7 @@ org.apache.maven.plugins maven-compiler-plugin - 3.11.0 + 3.12.1 11 @@ -96,7 +96,7 @@ org.apache.maven.plugins maven-source-plugin - 3.2.1 + 3.3.0 attach-sources @@ -109,7 +109,7 @@ org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.6.3 attach-javadocs From 969b7e7db24e61af3fc0c0d252266aa39e1ce4e3 Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Tue, 12 Mar 2024 16:19:47 +0800 Subject: [PATCH 038/149] CA-389840: Bug in parsing output of 'xen-livepatch list' The bug is in parsing the output of 'xen-livepatch list'. The following output can't be parsed correctly: lp_4.17.3-3.11.gf717213.xs8-4.17.3-3.12.xs8| CHECKED The regex pattern doesn't exclude the '|' after 'xs8'. As a result, the vertical line '|' is parsed as part of '3.12.xs8|'. The correct one should be '3.12.xs8'. Signed-off-by: Ming Lu --- ocaml/tests/test_livepatch.ml | 32 ++++++++++++++++++++++++++ ocaml/xapi/livepatch.ml | 43 ++++++++++++++++++----------------- 2 files changed, 54 insertions(+), 21 deletions(-) diff --git a/ocaml/tests/test_livepatch.ml b/ocaml/tests/test_livepatch.ml index 31b0eef8bfc..b87b657c8fe 100644 --- a/ocaml/tests/test_livepatch.ml +++ b/ocaml/tests/test_livepatch.ml @@ -52,6 +52,38 @@ lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8 | CHECKED |} , None ) + ; ( {| + ID | status +----------------------------------------+------------ +lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8| CHECKED +lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8| APPLIED + |} + , Some ("4.13.4", "10.22.xs8", "4.13.4", "10.23.xs8") + ) + ; ( {| + ID | status +----------------------------------------+------------ + lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8|CHECKED + lp_4.13.4-10.22.xs8-4.13.4-10.23.xs8|APPLIED + |} + , Some ("4.13.4", "10.22.xs8", "4.13.4", "10.23.xs8") + ) + ; ( {| + ID | status +----------------------------------------+------------ +p_4.13.4-10.22.xs8-4.13.4-10.23.xs8 | CHECKED +p_4.13.4-10.22.xs8-4.13.4-10.23.xs8 | APPLIED + |} + , None + ) + ; ( {| + ID | status | metadata +----------------------------------------+------------+--------------- +lp_4.17.3-3.11.gf717213.xs8-4.17.3-3.12.xs8| CHECKED | +lp_4.17.3-3.11.gf717213.xs8-4.17.3-3.13.xs8| APPLIED | + |} + , Some ("4.17.3", "3.11.gf717213.xs8", "4.17.3", "3.13.xs8") + ) ] end) diff --git a/ocaml/xapi/livepatch.ml b/ocaml/xapi/livepatch.ml index 0dc3869338e..63afa9a2c82 100644 --- a/ocaml/xapi/livepatch.ml +++ b/ocaml/xapi/livepatch.ml @@ -187,29 +187,34 @@ module KernelLivePatch = struct end module XenLivePatch = struct - let get_regexp status = - Re.Posix.compile_pat - (Printf.sprintf {|^[ ]*lp_([^- ]+)-([^- ]+)-([^- ]+)-([^- ]+).+%s.*$|} - status - ) + let drop x = Astring.Char.Ascii.(is_control x || is_white x) - let get_livepatches pattern s = + let get_livepatches state s = + let pattern = + Re.Posix.compile_pat {|^lp_([^- ]+)-([^- ]+)-([^- ]+)-([^- ]+)$|} + in Astring.String.cuts ~sep:"\n" s |> List.filter_map (fun line -> - match Re.exec_opt pattern line with - | Some groups -> - let base_version = Re.Group.get groups 1 in - let base_release = Re.Group.get groups 2 in - let to_version = Re.Group.get groups 3 in - let to_release = Re.Group.get groups 4 in - Some (base_version, base_release, to_version, to_release) - | None -> + Astring.String.cuts ~sep:"|" line + |> List.map (Astring.String.trim ~drop) + |> function + | name :: state' :: _ when state' = state -> ( + match Re.exec_opt pattern name with + | Some groups -> + let base_version = Re.Group.get groups 1 in + let base_release = Re.Group.get groups 2 in + let to_version = Re.Group.get groups 3 in + let to_release = Re.Group.get groups 4 in + Some (base_version, base_release, to_version, to_release) + | None -> + None + ) + | _ -> None ) let get_running_livepatch' s = - let r = get_regexp "APPLIED" in - get_livepatches r s |> get_latest_livepatch + get_livepatches "APPLIED" s |> get_latest_livepatch let get_running_livepatch () = Helpers.call_script !Xapi_globs.xen_livepatch_cmd ["list"] @@ -217,13 +222,9 @@ module XenLivePatch = struct let get_checked_livepatches () = Helpers.call_script !Xapi_globs.xen_livepatch_cmd ["list"] - |> get_livepatches (get_regexp "CHECKED") + |> get_livepatches "CHECKED" let get_base_build_id () = - let drop x = - let open Astring.Char.Ascii in - is_control x || is_blank x || is_white x - in Helpers.call_script !Xapi_globs.xl_cmd ["info"; "build_id"] |> Astring.String.trim ~drop |> function From d50b592d1b98ef2837f7bd5a4b9bf7df4da3aedf Mon Sep 17 00:00:00 2001 From: Lunfan Zhang Date: Tue, 12 Mar 2024 07:13:30 -0400 Subject: [PATCH 039/149] CP-48430 Update the running_domains metrics to count the not paused state domains Signed-off-by: Lunfan Zhang --- ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml b/ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml index 4f696085d6d..3eb708d7a5c 100644 --- a/ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml +++ b/ocaml/xcp-rrdd/bin/rrdd/xcp_rrdd.ml @@ -361,9 +361,11 @@ let dss_loadavg () = ) ] -let count_running_domain domains = +let count_power_state_running_domains domains = List.fold_left - (fun count (dom, _, _) -> if dom.Xenctrl.running then count + 1 else count) + (fun count (dom, _, _) -> + if not dom.Xenctrl.paused then count + 1 else count + ) 0 domains let dss_hostload xc domains = @@ -386,7 +388,7 @@ let dss_hostload xc domains = ) 0 domains in - let running_domains = count_running_domain domains in + let running_domains = count_power_state_running_domains domains in let load_per_cpu = float_of_int load /. float_of_int pcpus in [ From ae318530b9b3dd6af3637d1b1a0c3ea8303e4a46 Mon Sep 17 00:00:00 2001 From: Luca Zhang Date: Thu, 14 Mar 2024 15:29:24 +0800 Subject: [PATCH 040/149] fix typos: priviledges -> privileges Signed-off-by: Luca Zhang --- ocaml/idl/datamodel.ml | 2 +- ocaml/xapi/xapi_session.ml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index 8f452b47069..c8fa2614150 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -136,7 +136,7 @@ module Session = struct call ~flags:[`Session] ~name:"change_password" ~doc: "Change the account password; if your session is authenticated with \ - root priviledges then the old_pwd is validated and the new_pwd is set \ + root privileges then the old_pwd is validated and the new_pwd is set \ regardless" ~params: [ diff --git a/ocaml/xapi/xapi_session.ml b/ocaml/xapi/xapi_session.ml index 455dcef9c55..221498d6f1b 100644 --- a/ocaml/xapi/xapi_session.ml +++ b/ocaml/xapi/xapi_session.ml @@ -1149,8 +1149,8 @@ let change_password ~__context ~old_pwd ~new_pwd = try (* CP-696: only change password if session has is_local_superuser bit set *) (* - CA-13567: If you have root priviledges then we do not authenticate old_pwd; right now, since we only - ever have root priviledges we just comment this out. + CA-13567: If you have root privileges then we do not authenticate old_pwd; right now, since we only + ever have root privileges we just comment this out. begin try From 683a7b99aab8f527b0cc0aa54193fc23ac36b11c Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Sun, 17 Mar 2024 12:14:44 +0000 Subject: [PATCH 041/149] Remove Log_reopen forkexec RPC Not used. It was never implemented. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/lib/fe.ml | 1 - 1 file changed, 1 deletion(-) diff --git a/ocaml/forkexecd/lib/fe.ml b/ocaml/forkexecd/lib/fe.ml index bbad59dbfd2..c18f234803a 100644 --- a/ocaml/forkexecd/lib/fe.ml +++ b/ocaml/forkexecd/lib/fe.ml @@ -24,7 +24,6 @@ type ferpc = | Exec | Execed of int | Finished of process_result - | Log_reopen | Dontwaitpid [@@deriving rpc] From 4f9513ed1398c0200797b7ce28f38687aa08a647 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Sun, 17 Mar 2024 12:18:10 +0000 Subject: [PATCH 042/149] Remove Cancel forkexec RPC Not used. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/lib/fe.ml | 1 - ocaml/forkexecd/src/child.ml | 2 -- 2 files changed, 3 deletions(-) diff --git a/ocaml/forkexecd/lib/fe.ml b/ocaml/forkexecd/lib/fe.ml index c18f234803a..1a176a62baa 100644 --- a/ocaml/forkexecd/lib/fe.ml +++ b/ocaml/forkexecd/lib/fe.ml @@ -20,7 +20,6 @@ type process_result = WEXITED of int | WSIGNALED of int | WSTOPPED of int type ferpc = | Setup of setup_cmd | Setup_response of setup_response - | Cancel | Exec | Execed of int | Finished of process_result diff --git a/ocaml/forkexecd/src/child.ml b/ocaml/forkexecd/src/child.ml index f6ede6c608d..197f3b91f65 100644 --- a/ocaml/forkexecd/src/child.ml +++ b/ocaml/forkexecd/src/child.ml @@ -49,8 +49,6 @@ let handle_fd_sock fd_sock state = let handle_comms_sock comms_sock state = let call = Fecomms.read_raw_rpc comms_sock in match call with - | Ok Fe.Cancel -> - debug "Cancel" ; raise Cancelled | Ok Fe.Exec -> debug "Exec" ; {state with finished= true} From fbdf7331aa7747a9779930bcdac7968c6cc4b62a Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 14 Mar 2024 14:27:50 +0000 Subject: [PATCH 043/149] CA-390109: Use `$PROFILE` path to store and read known cert list Before these changes, the `SaveCertificates` method relied on machines having a `SpecialFolder.MyDocuments` folder. This is true in Windows and GUI versions of some Linux distros, but it's an assumption that caused the save method to fail if the folder didn't exist. With this commit, we're storing and reading from the path where `$PROFILE` is stored, which is platform agnostic from the point of view of the SDK. Signed-off-by: Danilo Del Busso --- .../autogen/Initialize-Environment.ps1 | 2 ++ .../autogen/src/CommonCmdletFunctions.cs | 29 +++++++++++++------ .../autogen/src/Connect-XenServer.cs | 8 ++--- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/ocaml/sdk-gen/powershell/autogen/Initialize-Environment.ps1 b/ocaml/sdk-gen/powershell/autogen/Initialize-Environment.ps1 index d418745ee39..c0d7b30dce3 100644 --- a/ocaml/sdk-gen/powershell/autogen/Initialize-Environment.ps1 +++ b/ocaml/sdk-gen/powershell/autogen/Initialize-Environment.ps1 @@ -46,4 +46,6 @@ if (Test-Path $perUserXsProfile) { Remove-Item variable:systemWideXsProfile Remove-Item variable:perUserXsProfile +$global:KnownServerCertificatesFilePath = Join-Path -Path (Split-Path $PROFILE) -ChildPath "XenServer_Known_Certificates.xml" + $XenServer_Environment_Initialized = $true diff --git a/ocaml/sdk-gen/powershell/autogen/src/CommonCmdletFunctions.cs b/ocaml/sdk-gen/powershell/autogen/src/CommonCmdletFunctions.cs index d01a03098cb..8f29ecde1f5 100644 --- a/ocaml/sdk-gen/powershell/autogen/src/CommonCmdletFunctions.cs +++ b/ocaml/sdk-gen/powershell/autogen/src/CommonCmdletFunctions.cs @@ -42,8 +42,10 @@ namespace Citrix.XenServer class CommonCmdletFunctions { private const string SessionsVariable = "global:Citrix.XenServer.Sessions"; + private const string DefaultSessionVariable = "global:XenServer_Default_Session"; - private static string CertificatePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"WindowsPowerShell\XenServer_Known_Certificates.xml"); + + private const string KnownServerCertificatesFilePathVariable = "global:KnownServerCertificatesFilePath"; static CommonCmdletFunctions() { @@ -68,8 +70,7 @@ internal static void SetAllSessions(PSCmdlet cmdlet, Dictionary internal static Session GetDefaultXenSession(PSCmdlet cmdlet) { - object obj = cmdlet.SessionState.PSVariable.GetValue(DefaultSessionVariable); - return obj as Session; + return cmdlet.SessionState.PSVariable.GetValue(DefaultSessionVariable) as Session; } internal static void SetDefaultXenSession(PSCmdlet cmdlet, Session session) @@ -77,19 +78,28 @@ internal static void SetDefaultXenSession(PSCmdlet cmdlet, Session session) cmdlet.SessionState.PSVariable.Set(DefaultSessionVariable, session); } + internal static string GetKnownServerCertificatesFilePathVariable(PSCmdlet cmdlet) + { + var knownCertificatesFilePathObject = cmdlet.SessionState.PSVariable.GetValue(KnownServerCertificatesFilePathVariable); + if (knownCertificatesFilePathObject is PSObject psObject) + return psObject.BaseObject as string; + return knownCertificatesFilePathObject?.ToString() ?? string.Empty; + } + internal static string GetUrl(string hostname, int port) { return string.Format("{0}://{1}:{2}", port == 80 ? "http" : "https", hostname, port); } - public static Dictionary LoadCertificates() + public static Dictionary LoadCertificates(PSCmdlet cmdlet) { Dictionary certificates = new Dictionary(); + var knownServerCertificatesFilePath = GetKnownServerCertificatesFilePathVariable(cmdlet); - if (File.Exists(CertificatePath)) + if (File.Exists(knownServerCertificatesFilePath)) { XmlDocument doc = new XmlDocument(); - doc.Load(CertificatePath); + doc.Load(knownServerCertificatesFilePath); foreach (XmlNode node in doc.GetElementsByTagName("certificate")) { @@ -104,9 +114,10 @@ public static Dictionary LoadCertificates() return certificates; } - public static void SaveCertificates(Dictionary certificates) + public static void SaveCertificates(PSCmdlet cmdlet, Dictionary certificates) { - string dirName = Path.GetDirectoryName(CertificatePath); + var knownServerCertificatesFilePath = GetKnownServerCertificatesFilePathVariable(cmdlet); + string dirName = Path.GetDirectoryName(knownServerCertificatesFilePath); if (!Directory.Exists(dirName)) Directory.CreateDirectory(dirName); @@ -129,7 +140,7 @@ public static void SaveCertificates(Dictionary certificates) } doc.AppendChild(node); - doc.Save(CertificatePath); + doc.Save(knownServerCertificatesFilePath); } public static string FingerprintPrettyString(string fingerprint) diff --git a/ocaml/sdk-gen/powershell/autogen/src/Connect-XenServer.cs b/ocaml/sdk-gen/powershell/autogen/src/Connect-XenServer.cs index 0ec80444a85..c1155f7a2a3 100644 --- a/ocaml/sdk-gen/powershell/autogen/src/Connect-XenServer.cs +++ b/ocaml/sdk-gen/powershell/autogen/src/Connect-XenServer.cs @@ -253,9 +253,9 @@ protected override void ProcessRecord() private void AddCertificate(string hostname, string fingerprint) { - var certificates = CommonCmdletFunctions.LoadCertificates(); + var certificates = CommonCmdletFunctions.LoadCertificates(this); certificates[hostname] = fingerprint; - CommonCmdletFunctions.SaveCertificates(certificates); + CommonCmdletFunctions.SaveCertificates(this, certificates); } private bool ValidateServerCertificate(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors) @@ -274,7 +274,7 @@ private bool ValidateServerCertificate(object sender, X509Certificate certificat bool trusted = VerifyInAllStores(new X509Certificate2(certificate)); - var certificates = CommonCmdletFunctions.LoadCertificates(); + var certificates = CommonCmdletFunctions.LoadCertificates(this); if (certificates.ContainsKey(hostname)) { @@ -292,7 +292,7 @@ private bool ValidateServerCertificate(object sender, X509Certificate certificat } certificates[hostname] = fingerprint; - CommonCmdletFunctions.SaveCertificates(certificates); + CommonCmdletFunctions.SaveCertificates(this, certificates); return true; } } From 8af3aa54e2dba2ef0711994ed8dbd10e69a85f42 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 14 Mar 2024 14:31:25 +0000 Subject: [PATCH 044/149] Fix typo in `XenServerPowerShell.csproj` Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj b/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj index 85ea0dc72b4..23fff01346e 100644 --- a/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj +++ b/ocaml/sdk-gen/powershell/autogen/src/XenServerPowerShell.csproj @@ -6,7 +6,7 @@ True - + true From 79ad453b6a45217c297715f607789e26e867a846 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 19 Mar 2024 14:50:45 +0000 Subject: [PATCH 045/149] message-switch: Print more complete time info in diagnostics Previously sub-second durations were printed as "ago", now they are pretty-printed using Mtime.Span's logic, for example "666ms ago". If the point in time is unexpectedly in the future, print "0ns ago". Durations lasting at least 1 seconds are printed as before. Signed-off-by: Pau Ruiz Safont --- ocaml/message-switch/cli/dune | 1 + ocaml/message-switch/cli/main.ml | 53 ++++++++++++++++++++++---------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/ocaml/message-switch/cli/dune b/ocaml/message-switch/cli/dune index beb3741dc85..c0741e71348 100644 --- a/ocaml/message-switch/cli/dune +++ b/ocaml/message-switch/cli/dune @@ -5,6 +5,7 @@ cmdliner message-switch-core message-switch-unix + mtime rpclib.core rpclib.json threads.posix diff --git a/ocaml/message-switch/cli/main.ml b/ocaml/message-switch/cli/main.ml index 197061a17ea..19324a5a25f 100644 --- a/ocaml/message-switch/cli/main.ml +++ b/ocaml/message-switch/cli/main.ml @@ -76,23 +76,36 @@ let help = ; `P (Printf.sprintf "Check bug reports at %s" project_url) ] +(* Durations, in nanoseconds *) +let second = 1_000_000_000L + +let minute = 60_000_000_000L + +let hour = 3600_000_000_000L + +let day = 86400_000_000_000L + (* Commands *) let diagnostics common_opts = Client.connect ~switch:common_opts.Common.path () >>|= fun t -> Client.diagnostics ~t () >>|= fun d -> let open Message_switch_core.Protocol in - let in_the_past = Int64.sub d.Diagnostics.current_time in + let in_the_past ts = + if d.Diagnostics.current_time < ts then + 0L + else + Int64.sub d.Diagnostics.current_time ts + in let time f x = - let open Int64 in - let secs = div (f x) 1_000_000_000L in - let secs' = rem secs 60L in - let mins = div secs 60L in - let mins' = rem mins 60L in - let hours = div mins 60L in - let hours' = rem hours 24L in - let days = div hours 24L in - let fragment name = function + let timespan = f x in + let ( // ) = Int64.div in + let ( %% ) = Int64.rem in + let secs = timespan %% minute // second in + let mins = timespan %% hour // minute in + let hours = timespan %% day // hour in + let days = timespan // day in + let format name = function | 0L -> [] | 1L -> @@ -101,11 +114,10 @@ let diagnostics common_opts = [Printf.sprintf "%Ld %ss" n name] in let bits = - fragment "day" days - @ fragment "hour" hours' - @ fragment "min" mins' - @ fragment "second" secs' - @ [] + format "day" days + @ format "hour" hours + @ format "min" mins + @ format "second" secs in let length = List.length bits in let _, rev_bits = @@ -122,7 +134,16 @@ let diagnostics common_opts = ) (0, []) bits in - String.concat "" (List.rev rev_bits) ^ "ago" + let format_secs ts = + Mtime.Span.(Format.asprintf "%a " pp (of_uint64_ns ts)) + in + let timestrings = + if rev_bits = [] then + [format_secs (timespan %% minute)] + else + List.rev rev_bits + in + String.concat "" timestrings ^ "ago" in let origin = function | Anonymous id -> From e15763d1d8b51e53c233541471a8783e363147fc Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Mon, 25 Mar 2024 14:10:04 +0800 Subject: [PATCH 046/149] CA-390570: Py3 socket.sendto needs bytes instead of a string Signed-off-by: Ming Lu --- scripts/plugins/perfmon | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/plugins/perfmon b/scripts/plugins/perfmon index 2186c938938..e3dc2452691 100644 --- a/scripts/plugins/perfmon +++ b/scripts/plugins/perfmon @@ -14,16 +14,17 @@ def send_perfmon_cmd(cmd): "Return True for success, or ERROR_%d: otherwise" if len(cmd) >= cmdmaxlen: return "ERROR_0: command too long" + cmd_bytes = cmd.encode() try: sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) - rc = sock.sendto(cmd, cmdsockname) + rc = sock.sendto(cmd_bytes, cmdsockname) except socket.error as e: err, msg = e.args return "ERROR_%d: %s" % (err, msg) except Exception: return "ERROR_1: unknown error" - return str(rc == len(cmd)) + return str(rc == len(cmd_bytes)) def stop(session, args): From bbc5a1f7988d47408cb63e56e2cbdaf74f7529a7 Mon Sep 17 00:00:00 2001 From: Christian Lindig Date: Thu, 1 Feb 2024 10:24:53 +0000 Subject: [PATCH 047/149] CP-46179 create backup VDI with determinstic UUID When creating a backup VDI on an SR we want to derive the VDI's UUID from the SR. This way we can't be later tricked into accepting a different VDI as a backup. Implement a hash that derives the VDI UUID and pass this UUID to SM to be used. Currently the creation of this VDI in Xapi is detected based on its name-label: "Pool Metadata Backup". The SM stack usually creates a new random UUID but will use a given UUID passed as "vdi_uuid" for its "vdi_create" command. The difficulty in the implementation is that "vdi_uuid" is used in other commands as well and we have to make sure to set it only in the intended context. It is not straight forward to pass the UUID from Xapi_vdi.create down to Sm. Signed-off-by: Christian Lindig --- ocaml/libs/uuid/uuidx.ml | 12 ++++++++++++ ocaml/libs/uuid/uuidx.mli | 8 ++++++++ ocaml/xapi/sm.ml | 6 +++--- ocaml/xapi/sm_exec.ml | 21 ++++++++++++++++++--- ocaml/xapi/storage_smapiv1.ml | 26 ++++++++++++++++++++------ ocaml/xapi/xapi_vdi.ml | 16 +++++++++++++++- 6 files changed, 76 insertions(+), 13 deletions(-) diff --git a/ocaml/libs/uuid/uuidx.ml b/ocaml/libs/uuid/uuidx.ml index de471da0e1b..01dbda46899 100644 --- a/ocaml/libs/uuid/uuidx.ml +++ b/ocaml/libs/uuid/uuidx.ml @@ -83,3 +83,15 @@ let string_of_uuid = to_string let uuid_of_int_array = of_int_array let int_array_of_uuid = to_int_array + +module Hash = struct + (** Derive a deterministic UUID from a string: the same + string maps to the same UUID. We are using our own namespace; the + namespace is not a secret *) + + let namespace = + let ns = "e93e0639-2bdb-4a59-8b46-352b3f408c19" in + Uuidm.(of_string ns |> Option.get) + + let string str = Uuidm.v5 namespace str +end diff --git a/ocaml/libs/uuid/uuidx.mli b/ocaml/libs/uuid/uuidx.mli index 57b4058b8ca..618235b4ae6 100644 --- a/ocaml/libs/uuid/uuidx.mli +++ b/ocaml/libs/uuid/uuidx.mli @@ -81,3 +81,11 @@ val make_cookie : unit -> cookie val cookie_of_string : string -> cookie val string_of_cookie : cookie -> string + +module Hash : sig + (** hash a string (deterministically) into a UUID. This uses + namespace UUID e93e0639-2bdb-4a59-8b46-352b3f408c19. *) + + (* UUID Version 5 derived from argument string and namespace UUID *) + val string : string -> 'a t +end diff --git a/ocaml/xapi/sm.ml b/ocaml/xapi/sm.ml index 2526aa79b6c..0ae899f63f0 100644 --- a/ocaml/xapi/sm.ml +++ b/ocaml/xapi/sm.ml @@ -152,7 +152,7 @@ let sr_update ~dbg dconf driver sr = let call = Sm_exec.make_call ~sr_ref:sr dconf "sr_update" [] in Sm_exec.parse_unit (Sm_exec.exec_xmlrpc ~dbg (driver_filename driver) call) -let vdi_create ~dbg dconf driver sr sm_config vdi_type size name_label +let vdi_create ~dbg ?vdi_uuid dconf driver sr sm_config vdi_type size name_label name_description metadata_of_pool is_a_snapshot snapshot_time snapshot_of read_only = with_dbg ~dbg ~name:"vdi_create" @@ fun di -> @@ -164,8 +164,8 @@ let vdi_create ~dbg dconf driver sr sm_config vdi_type size name_label ) ; srmaster_only dconf ; let call = - Sm_exec.make_call ~sr_ref:sr ~vdi_sm_config:sm_config ~vdi_type dconf - "vdi_create" + Sm_exec.make_call ?vdi_uuid ~sr_ref:sr ~vdi_sm_config:sm_config ~vdi_type + dconf "vdi_create" [ sprintf "%Lu" size ; name_label diff --git a/ocaml/xapi/sm_exec.ml b/ocaml/xapi/sm_exec.ml index a6d0d231ee2..a55b61d72f9 100644 --- a/ocaml/xapi/sm_exec.ml +++ b/ocaml/xapi/sm_exec.ml @@ -69,8 +69,8 @@ type call = { } let make_call ?driver_params ?sr_sm_config ?vdi_sm_config ?vdi_type - ?vdi_location ?new_uuid ?sr_ref ?vdi_ref (subtask_of, device_config) cmd - args = + ?vdi_location ?new_uuid ?sr_ref ?vdi_ref ?vdi_uuid + (subtask_of, device_config) cmd args = Server_helpers.exec_with_new_task "sm_exec" (fun __context -> (* Only allow a subset of calls if the SR has been introduced by a DR task. *) Option.iter @@ -117,7 +117,22 @@ let make_call ?driver_params ?sr_sm_config ?vdi_sm_config ?vdi_type Option.map (fun self -> Db.VDI.get_location ~__context ~self) vdi_ref in let vdi_uuid = - Option.map (fun self -> Db.VDI.get_uuid ~__context ~self) vdi_ref + match (cmd, vdi_ref, vdi_uuid) with + | "vdi_create", None, (Some x as uuid) -> + debug "%s: cmd=%s vdi_uuid=%s" __FUNCTION__ cmd x ; + uuid + (* when creating a VDI we sometimes want to provide the UUID + rather than letting the backend pick one. This is to + support backup VDIs CP-46179. So in that case, use the + provided UUID but not for other commands *) + | _, None, Some uuid -> + warn "%s: cmd=%s vdi_uuid=%s - should not happen" __FUNCTION__ cmd + uuid ; + None + | _, Some self, _ -> + Db.VDI.get_uuid ~__context ~self |> Option.some + | _, None, None -> + None in let vdi_on_boot = Option.map diff --git a/ocaml/xapi/storage_smapiv1.ml b/ocaml/xapi/storage_smapiv1.ml index 9ca3660eeb6..c7bdd772a28 100644 --- a/ocaml/xapi/storage_smapiv1.ml +++ b/ocaml/xapi/storage_smapiv1.ml @@ -691,19 +691,33 @@ module SMAPIv1 : Server_impl = struct let uuid = require_uuid vi in vdi_info_from_db ~__context (Db.VDI.get_by_uuid ~__context ~uuid) - let create _context ~dbg ~sr ~vdi_info = + let create _context ~dbg ~sr ~(vdi_info : Storage_interface.vdi_info) = with_dbg ~name:"VDI.create" ~dbg @@ fun di -> let dbg = Debuginfo.to_string di in try Server_helpers.exec_with_new_task "VDI.create" ~subtask_of:(Ref.of_string dbg) (fun __context -> - let sr = Db.SR.get_by_uuid ~__context ~uuid:(s_of_sr sr) in + let sr_uuid = s_of_sr sr in + let sr = Db.SR.get_by_uuid ~__context ~uuid:sr_uuid in let vi = + (* we want to set vdi_uuid when creating a backup VDI with + a specific UUID. SM picks up vdi_uuid instead of creating + a new random UUID; Cf. Xapi_vdi.create *) + let vdi_uuid = + match vdi_info.uuid with + | Some uuid when uuid = Uuidx.(Hash.string sr_uuid |> to_string) + -> + info "%s: creating a backup VDI %s" __FUNCTION__ uuid ; + vdi_info.uuid + | _ -> + None + in Sm.call_sm_functions ~__context ~sR:sr (fun device_config _type -> - Sm.vdi_create ~dbg device_config _type sr vdi_info.sm_config - vdi_info.ty vdi_info.virtual_size vdi_info.name_label - vdi_info.name_description vdi_info.metadata_of_pool - vdi_info.is_a_snapshot vdi_info.snapshot_time + Sm.vdi_create ~dbg ?vdi_uuid device_config _type sr + vdi_info.sm_config vdi_info.ty vdi_info.virtual_size + vdi_info.name_label vdi_info.name_description + vdi_info.metadata_of_pool vdi_info.is_a_snapshot + vdi_info.snapshot_time (s_of_vdi vdi_info.snapshot_of) vdi_info.read_only ) diff --git a/ocaml/xapi/xapi_vdi.ml b/ocaml/xapi/xapi_vdi.ml index ac989551b85..bc7bafb23d0 100644 --- a/ocaml/xapi/xapi_vdi.ml +++ b/ocaml/xapi/xapi_vdi.ml @@ -625,13 +625,27 @@ let create ~__context ~name_label ~name_description ~sR ~virtual_size ~_type | `cbt_metadata -> "cbt_metadata" in + (* special case: we want to use a specific UUID for Pool Meta Data + Backup *) + let uuid_ = + match (_type, name_label) with + | `user, "Pool Metadata Backup" -> + let sr = Db.SR.get_uuid ~__context ~self:sR in + let uuid = Uuidx.(Hash.string sr |> to_string) in + info "%s: using deterministic UUID for '%s' VDI: %s" __FUNCTION__ + name_label uuid ; + Some uuid + | _ -> + None + in let open Storage_access in let task = Context.get_task_id __context in let open Storage_interface in let vdi_info = { Storage_interface.default_vdi_info with - name_label + uuid= uuid_ + ; name_label ; name_description ; ty= vdi_type ; read_only From 65f152de687229946eaea6ddcaa5e3d0a11b2b01 Mon Sep 17 00:00:00 2001 From: Christian Lindig Date: Fri, 9 Feb 2024 15:21:05 +0000 Subject: [PATCH 048/149] CP-46179 update metadata backup/restore to use deterministic UUID We introduce deterministic UUID for backup VDIs to avoid searching for VDIs. Add this capability to the scripts that create and restore backups. In case the VDI with the expected UUID does not exist, let the user confirm probing for it. Signed-off-by: Christian Lindig --- scripts/xe-backup-metadata | 44 +++++++++++++++++++++++++++++++++---- scripts/xe-restore-metadata | 44 +++++++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/scripts/xe-backup-metadata b/scripts/xe-backup-metadata index 4aa09b7f74c..c67f04a0b4c 100755 --- a/scripts/xe-backup-metadata +++ b/scripts/xe-backup-metadata @@ -48,6 +48,29 @@ function usage { exit 1 } +function uuid5 { + # could use a modern uuidgen but it's not on XS 8 + # should work with Python 2 and 3 + python -c "import uuid; print (uuid.uuid5(uuid.UUID('$1'), '$2'))" +} + +function validate_vdi_uuid { + # we check that vdi has the expected UUID which depends on the UUID of + # the SR. This is a deterministic hash of the SR UUID and the + # namespace UUID $NS. This UUID must match what Xapi's Uuidx module is using. + local NS="e93e0639-2bdb-4a59-8b46-352b3f408c19" + local sr="$1" + local vdi="$2" + local uuid + + uuid=$(uuid5 "$NS" "$sr") + if [ "$vdi" != "$uuid" ]; then + return 1 + else + return 0 + fi +} + function test_sr { sr_uuid_found=$(${XE} sr-list uuid="$1" --minimal) if [ "${sr_uuid_found}" != "$1" ]; then @@ -93,7 +116,7 @@ if [ -z "${sr_uuid}" ]; then fi test_sr "${sr_uuid}" -sr_name=$(${XE} sr-param-get uuid=${sr_uuid} param-name=name-label) +sr_name=$(${XE} sr-param-get uuid="${sr_uuid}" param-name=name-label) # see if a backup VDI already exists on the selected SR vdi_uuid=$(${XE} vdi-list other-config:ctxs-pool-backup=true sr-uuid=${sr_uuid} params=uuid --minimal) @@ -134,11 +157,24 @@ function cleanup { fi } -echo Using SR: ${sr_name} +# if we can't validate the UUID of the VDI, prompt the user +if [ -n "${vdi_uuid}" ]; then + if ! validate_vdi_uuid "${sr_uuid}" "${vdi_uuid}"; then + echo "Backup VDI $vdi_uuid was most likley create by an earlier" + echo "version of this code. Make sure this is a VDI that you" + echo "created as we can't validate it without mounting it." + read -p "Continue? [Y/N]" -n 1 -r; echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi + fi +fi + +echo "Using SR: ${sr_name}" if [ -z "${vdi_uuid}" ]; then if [ "${create_vdi}" -gt 0 ]; then echo -n "Creating new backup VDI: " - vdi_uuid=$(${XE} vdi-create virtual-size=500MiB sr-uuid=${sr_uuid} type=user name-label="Pool Metadata Backup") + label="Pool Metadata Backup" + # the label must match what xapi_vdi.ml is using for backup VDIs + vdi_uuid=$(${XE} vdi-create virtual-size=500MiB sr-uuid="${sr_uuid}" type=user name-label="${label}") init_fs=1 if [ $? -ne 0 ]; then echo failed @@ -225,7 +261,7 @@ if [ ${leave_mounted} -eq 0 ]; then usage=`cd ${mnt} && df . | sed -n "2p" | awk '{ print $5 }' | tr -d '%'` echo "Checking backup VDI space usage: $usage%" if [ $usage -gt $usage_alert ] && [ ${force_backup} -eq 0 ]; then - echo "Running out of space, you can use "-d" option to attach VDI and free more space, exit now." + echo "Running out of space, you can use '-d' option to attach VDI and free more space, exit now." cleanup exit 1 fi diff --git a/scripts/xe-restore-metadata b/scripts/xe-restore-metadata index 81beb51b704..6c2018ccb54 100755 --- a/scripts/xe-restore-metadata +++ b/scripts/xe-restore-metadata @@ -52,6 +52,13 @@ function test_sr { fi } +# name space to hash SRs for a deterministic VDI UUID +NS="e93e0639-2bdb-4a59-8b46-352b3f408c19" +function uuid5 { + # could use a modern uuidgen but it's not on XS 8 + python -c "import uuid; print (uuid.uuid5(uuid.UUID('$1'), '$2'))" +} + dry_run=0 sr_uuid= yes=0 @@ -100,9 +107,16 @@ test_sr "${sr_uuid}" sr_name=$(${XE} sr-param-get uuid=${sr_uuid} param-name=name-label) +# probe first for a VDI with known UUID derived from the SR to avoid +# scanning for a VDI +backup_vdi=$(uuid5 "${NS}" "${sr_uuid}") +if [ -z "${vdis}" ]; then + vdis=$(${XE} vdi-list uuid="${backup_vdi}" sr-uuid="${sr_uuid}" read-only=false --minimal) +fi + # get a list of all VDIs if an override has not been provided on the cmd line if [ -z "${vdis}" ]; then - vdis=$(${XE} vdi-list params=uuid sr-uuid=${sr_uuid} read-only=false --minimal) + vdis=$(${XE} vdi-list params=uuid sr-uuid="${sr_uuid}" read-only=false --minimal) fi mnt= @@ -149,6 +163,14 @@ fi trap cleanup SIGINT ERR for vdi_uuid in ${vdis}; do + if [ "${vdi_uuid}" != "${backup_vdi}" ]; then + echo "Probing VDI ${vdi_uuid}." + echo "This VDI was created with a prior version of XenServer." + echo "Its validity can't be checked without mounting it first." + read -p "Continue? [Y/N]" -n 1 -r; echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi + fi + ${debug} echo -n "Creating VBD: " >&2 vbd_uuid=$(${XE} vbd-create vm-uuid=${CONTROL_DOMAIN_UUID} vdi-uuid=${vdi_uuid} device=autodetect 2>/dev/null) @@ -253,7 +275,7 @@ all) ;; esac -if [ ! -d ${full_dir} ]; then +if [ ! -d "${full_dir}" ]; then echo No VM metadata exports were found for the selected SR >&2 cleanup exit 1 @@ -266,9 +288,9 @@ ${debug} echo "" >&2 ${debug} echo Latest VM metadata found is: >&2 ${debug} ls >&2 -if [ $yes -eq 0 ]; then +if [ "$yes" -eq 0 ]; then echo "Do you wish to reimport all VM metadata?" - echo "Please type in "yes" and to continue." + echo "Please type in 'yes' and to continue." read response if [ "$response" != "yes" ]; then echo Aborting metadata restore. @@ -297,8 +319,8 @@ else fi shopt -s nullglob for meta in *.vmmeta; do - echo xe vm-import filename=${meta} sr-uuid=${sr_uuid} --metadata --preserve${force_flag}${dry_run_flag} - "@OPTDIR@/bin/xe" vm-import filename="${full_dir}/${meta}" sr-uuid=${sr_uuid} --metadata --preserve${force_flag}${dry_run_flag} + echo xe vm-import filename="${meta}" sr-uuid="${sr_uuid}" --metadata --preserve"${force_flag}""${dry_run_flag}" + "@OPTDIR@/bin/xe" vm-import filename="${full_dir}/${meta}" sr-uuid="${sr_uuid}" --metadata --preserve"${force_flag}""${dry_run_flag}" if [ $? -gt 0 ]; then error_count=$(( $error_count + 1 )) else @@ -306,16 +328,16 @@ for meta in *.vmmeta; do fi done -smmeta_file=${mnt}/metadata/${chosen_metadata_dir}/SRMETA.xml +smmeta_file="${mnt}/metadata/${chosen_metadata_dir}/SRMETA.xml" if [ "$restore_mode" == "all" ]; then cmd="@LIBEXECDIR@/restore-sr-metadata.py -f ${smmeta_file}" else cmd="@LIBEXECDIR@/restore-sr-metadata.py -u ${sr_uuid} -f ${smmeta_file}" fi -if [ -e ${smmeta_file} ]; then - if [ ${dry_run} -gt 0 ]; then - echo ${cmd} +if [ -e "${smmeta_file}" ]; then + if [ "${dry_run}" -gt 0 ]; then + echo "${cmd}" else ${cmd} fi @@ -323,4 +345,4 @@ fi echo "Restored ${good_count} VM(s), and ${error_count} error(s)" cleanup -exit ${error_count} +exit "${error_count}" From c2f7456718f059d1d262d5de521fb447a203f47a Mon Sep 17 00:00:00 2001 From: Christian Lindig Date: Thu, 21 Mar 2024 14:38:59 +0000 Subject: [PATCH 049/149] CP-46179 improve quoting in shell code Signed-off-by: Christian Lindig --- scripts/xe-backup-metadata | 106 ++++++++++++++++++------------------ scripts/xe-restore-metadata | 96 ++++++++++++++++---------------- 2 files changed, 101 insertions(+), 101 deletions(-) diff --git a/scripts/xe-backup-metadata b/scripts/xe-backup-metadata index c67f04a0b4c..d6dc9b877f9 100755 --- a/scripts/xe-backup-metadata +++ b/scripts/xe-backup-metadata @@ -74,7 +74,7 @@ function validate_vdi_uuid { function test_sr { sr_uuid_found=$(${XE} sr-list uuid="$1" --minimal) if [ "${sr_uuid_found}" != "$1" ]; then - echo Invalid SR UUID specified: $1 + echo "Invalid SR UUID specified: $1" usage fi } @@ -112,32 +112,32 @@ fi # determine if the SR UUID is vaid if [ -z "${sr_uuid}" ]; then # use the default-SR from the pool - sr_uuid=$(${XE} pool-param-get uuid=${pool_uuid} param-name=default-SR) + sr_uuid=$(${XE} pool-param-get uuid="${pool_uuid}" param-name=default-SR) fi test_sr "${sr_uuid}" sr_name=$(${XE} sr-param-get uuid="${sr_uuid}" param-name=name-label) # see if a backup VDI already exists on the selected SR -vdi_uuid=$(${XE} vdi-list other-config:ctxs-pool-backup=true sr-uuid=${sr_uuid} params=uuid --minimal) +vdi_uuid=$(${XE} vdi-list other-config:ctxs-pool-backup=true sr-uuid="${sr_uuid}" params=uuid --minimal) mnt= function cleanup { trap "" TERM INT cd / if [ ! -z "${mnt}" ]; then - umount ${mnt} >/dev/null 2>&1 - rmdir ${mnt} + umount "${mnt}" >/dev/null 2>&1 + rmdir "${mnt}" fi if [ ! -z "${vbd_uuid}" ]; then ${debug} echo -n "Unplugging VBD: " - ${XE} vbd-unplug uuid=${vbd_uuid} timeout=20 + ${XE} vbd-unplug uuid="${vbd_uuid}" timeout=20 # poll for the device to go away if we know its name if [ "${device}" != "" ]; then device_gone=0 for ((i=0; i<10; i++)); do ${debug} echo -n "." - if [ ! -b ${device} ]; then + if [ ! -b "${device}" ]; then ${debug} echo " done" device_gone=1 break @@ -146,14 +146,14 @@ function cleanup { done if [ ${device_gone} -eq 0 ]; then ${debug} echo " failed" - echo Please destroy VBD ${vbd_uuid} manually. + echo "Please destroy VBD ${vbd_uuid} manually." else - ${XE} vbd-destroy uuid=${vbd_uuid} + ${XE} vbd-destroy uuid="${vbd_uuid}" fi fi fi if [ ${fs_uninitialised} -eq 1 -a -n "${vdi_uuid}" ] ; then - ${XE} vdi-destroy uuid=${vdi_uuid} + ${XE} vdi-destroy uuid="${vdi_uuid}" fi } @@ -184,8 +184,8 @@ if [ -z "${vdi_uuid}" ]; then echo "Backup VDI not found, aborting. You can initialise one using the '-c' flag." exit 3 fi - echo ${vdi_uuid} - ${XE} vdi-param-set uuid=${vdi_uuid} other-config:ctxs-pool-backup=true + echo "${vdi_uuid}" + ${XE} vdi-param-set uuid="${vdi_uuid}" other-config:ctxs-pool-backup=true else ${debug} echo "Using existing backup VDI: ${vdi_uuid}" fs_uninitialised=0 @@ -196,71 +196,71 @@ if [ ${just_find_vdi} -gt 0 ]; then fi ${debug} echo -n "Creating VBD: " -vbd_uuid=$(${XE} vbd-create vm-uuid=${CONTROL_DOMAIN_UUID} vdi-uuid=${vdi_uuid} device=autodetect) -${debug} echo ${vbd_uuid} +vbd_uuid=$(${XE} vbd-create vm-uuid="${CONTROL_DOMAIN_UUID}" vdi-uuid="${vdi_uuid}" device=autodetect) +${debug} echo "${vbd_uuid}" if [ $? -ne 0 -o -z "${vbd_uuid}" ]; then - echo error creating VBD + echo "error creating VBD" cleanup exit 1 fi ${debug} echo -n "Plugging VBD: " -${XE} vbd-plug uuid=${vbd_uuid} -device=/dev/$(${XE} vbd-param-get uuid=${vbd_uuid} param-name=device) +${XE} vbd-plug uuid="${vbd_uuid}" +device=/dev/$(${XE} vbd-param-get uuid="${vbd_uuid}" param-name=device) -if [ ! -b ${device} ]; then - ${debug} echo ${device}: not a block special +if [ ! -b "${device}" ]; then + ${debug} echo "${device}: not a block special" cleanup exit 1 fi -${debug} echo ${device} +${debug} echo "${device}" -if [ $init_fs -eq 1 ]; then +if [ "$init_fs" -eq 1 ]; then ${debug} echo -n "Creating filesystem: " - mkfs.ext3 -j -F ${device} > /dev/null 2>&1 + mkfs.ext3 -j -F "${device}" > /dev/null 2>&1 ${debug} echo "done" fs_uninitialised=0 fi ${debug} echo -n "Mounting filesystem: " -mnt=/var/run/pool-backup-${vdi_uuid} -mkdir -p ${mnt} +mnt="/var/run/pool-backup-${vdi_uuid}" +mkdir -p "${mnt}" -/sbin/fsck -a ${device} >/dev/null 2>&1 +/sbin/fsck -a "${device}" >/dev/null 2>&1 if [ $? -ne 0 ]; then ${debug} fsck failed. Please correct manually cleanup exit 1 fi -mount ${device} ${mnt} > /dev/null 2>&1 +mount "${device}" "${mnt}" > /dev/null 2>&1 if [ $? -ne 0 ]; then ${debug} echo failed cleanup exit 1 fi -${debug} echo ${mnt} +${debug} echo "${mnt}" if [ ${leave_mounted} -eq 0 ]; then - lrconf=${mnt}/conf/${vdi_uuid} - if [ ! -f ${lrconf} ]; then + lrconf="${mnt}/conf/${vdi_uuid}" + if [ ! -f "${lrconf}" ]; then ${debug} echo -n "Initialising rotation: " - mkdir -p ${mnt}/conf/ - echo "${mnt}/${pool_uuid}.db {" >> ${lrconf} - echo " rotate ${history_kept}" >> ${lrconf} - echo " missingok" >> ${lrconf} - echo "}" >> ${lrconf} - echo done - echo ${metadata_version} >> ${mnt}/.ctxs-metadata-backup + mkdir -p "${mnt}/conf/" + echo "${mnt}/${pool_uuid}.db {" >> "${lrconf}" + echo " rotate ${history_kept}" >> "${lrconf}" + echo " missingok" >> "${lrconf}" + echo "}" >> "${lrconf}" + echo "done" + echo "${metadata_version}" >> "${mnt}/.ctxs-metadata-backup" fi # check the usage of the backup VDI - usage=`cd ${mnt} && df . | sed -n "2p" | awk '{ print $5 }' | tr -d '%'` + usage=`cd "${mnt}" && df . | sed -n "2p" | awk '{ print $5 }' | tr -d '%'` echo "Checking backup VDI space usage: $usage%" - if [ $usage -gt $usage_alert ] && [ ${force_backup} -eq 0 ]; then + if [ "$usage" -gt "$usage_alert" ] && [ "${force_backup}" -eq 0 ]; then echo "Running out of space, you can use '-d' option to attach VDI and free more space, exit now." cleanup exit 1 @@ -268,38 +268,38 @@ if [ ${leave_mounted} -eq 0 ]; then # invoke logrotate to rotate over old pool db backups echo -n "Rotating old backups: " - logrotate -f ${lrconf} - num_found=$(find ${mnt} -name \*.db\.* | wc -l) - echo found ${num_found} + logrotate -f "${lrconf}" + num_found=$(find "${mnt}" -name '*.db.*' | wc -l) + echo "found ${num_found}" # perform the pool database dump echo -n "Backing up pool database: " - ${XE} pool-dump-database file-name=${mnt}/${pool_uuid}.db + ${XE} pool-dump-database file-name="${mnt}/${pool_uuid}.db" echo done # backup the VM metadata for each VM in the pool into a dated directory datetime=$(date +%F-%H-%M-%S) - metadir=${mnt}/metadata/${datetime} - mkdir -p ${metadir} + metadir="${mnt}/metadata/${datetime}" + mkdir -p "${metadir}" echo -n "Cleaning old VM metadata: " IFS=" " - todelete=$(cd ${mnt}/metadata && ls -1 |sort -n | head -n -${history_kept} | xargs echo) + todelete=$(cd "${mnt}/metadata" && ls -1 |sort -n | head -n -${history_kept} | xargs echo) for dir in ${todelete}; do - rm -rf ${mnt}/metadata/${dir} + rm -rf "${mnt}/metadata/${dir}" done echo done IFS="," echo -n "Backing up SR metadata: " - mkdir -p ${metadir} - "@LIBEXECDIR@/backup-sr-metadata.py" -f ${metadir}/SRMETA.xml + mkdir -p "${metadir}" + "@LIBEXECDIR@/backup-sr-metadata.py" -f "${metadir}/SRMETA.xml" echo "done" echo -n "Backing up VM metadata: " ${debug} echo "" - mkdir -p ${metadir}/all + mkdir -p "${metadir}/all" for vmuuid in $(${XE} vm-list params=uuid is-control-domain=false --minimal); do ${debug} echo -n . - ${XE} vm-export --metadata uuid=${vmuuid} filename=${metadir}/all/${vmuuid}.vmmeta >/dev/null 2>&1 + ${XE} vm-export --metadata uuid="${vmuuid}" filename="${metadir}/all/${vmuuid}.vmmeta" >/dev/null 2>&1 done echo "done" echo -n "Backing up Template metadata: " @@ -307,13 +307,13 @@ if [ ${leave_mounted} -eq 0 ]; then template_uuids=$("@LIBEXECDIR@/print-custom-templates") if [ $? -eq 0 ]; then for tmpl_uuid in ${template_uuids}; do - ${XE} template-export --metadata template-uuid=${tmpl_uuid} filename=${metadir}/all/${tmpl_uuid}.vmmeta >/dev/null 2>&1 + ${XE} template-export --metadata template-uuid="${tmpl_uuid}" filename="${metadir}/all/${tmpl_uuid}.vmmeta" >/dev/null 2>&1 done fi echo "done" - "@LIBEXECDIR@/link-vms-by-sr.py" -d ${metadir} + "@LIBEXECDIR@/link-vms-by-sr.py" -d "${metadir}" else - cd ${mnt} + cd "${mnt}" env PS1="Mounted backup VDI on: ${mnt}\nPress ^D to exit shell and safely detach it.\n\n[\u@\h \W]\$ " bash fi diff --git a/scripts/xe-restore-metadata b/scripts/xe-restore-metadata index 6c2018ccb54..d9ff3d2d37e 100755 --- a/scripts/xe-restore-metadata +++ b/scripts/xe-restore-metadata @@ -47,7 +47,7 @@ function usage { function test_sr { sr_uuid_found=$(${XE} sr-list uuid="$1" --minimal) if [ "${sr_uuid_found}" != "$1" ]; then - echo Invalid SR UUID specified: $1 + echo "Invalid SR UUID specified: $1" usage fi } @@ -101,11 +101,11 @@ fi # determine if the SR UUID is vaid if [ -z "${sr_uuid}" ]; then # use the default-SR from the pool - sr_uuid=$(${XE} pool-param-get uuid=${pool_uuid} param-name=default-SR) + sr_uuid=$(${XE} pool-param-get uuid="${pool_uuid}" param-name=default-SR) fi test_sr "${sr_uuid}" -sr_name=$(${XE} sr-param-get uuid=${sr_uuid} param-name=name-label) +sr_name=$(${XE} sr-param-get uuid="${sr_uuid}" param-name=name-label) # probe first for a VDI with known UUID derived from the SR to avoid # scanning for a VDI @@ -123,20 +123,20 @@ mnt= function cleanup { cd / if [ ! -z "${mnt}" ]; then - umount ${mnt} >/dev/null 2>&1 - rmdir ${mnt} + umount "${mnt}" >/dev/null 2>&1 + rmdir "${mnt}" mnt="" fi if [ ! -z "${vbd_uuid}" ]; then ${debug} echo -n "Unplugging VBD: " >&2 - ${XE} vbd-unplug uuid=${vbd_uuid} timeout=20 + ${XE} vbd-unplug uuid="${vbd_uuid}" timeout=20 # poll for the device to go away if we know its name if [ "${device}" != "" ]; then device_gone=0 for ((i=0; i<10; i++)); do ${debug} echo -n "." >&2 - if [ ! -b ${device} ]; then + if [ ! -b "${device}" ]; then ${debug} echo " done" >&2 device_gone=1 break @@ -145,9 +145,9 @@ function cleanup { done if [ ${device_gone} -eq 0 ]; then ${debug} echo " failed" >&2 - ${debug} echo Please destroy VBD ${vbd_uuid} manually. >&2 + ${debug} echo "Please destroy VBD ${vbd_uuid} manually." >&2 else - ${XE} vbd-destroy uuid=${vbd_uuid} + ${XE} vbd-destroy uuid="${vbd_uuid}" vbd_uuid="" fi fi @@ -156,7 +156,7 @@ function cleanup { } if [ -z "${vdis}" ]; then - echo No VDIs found on SR. >&2 + echo "No VDIs found on SR." >&2 exit 0 fi @@ -165,87 +165,87 @@ trap cleanup SIGINT ERR for vdi_uuid in ${vdis}; do if [ "${vdi_uuid}" != "${backup_vdi}" ]; then echo "Probing VDI ${vdi_uuid}." - echo "This VDI was created with a prior version of XenServer." + echo "This VDI was created with a prior version of this code." echo "Its validity can't be checked without mounting it first." read -p "Continue? [Y/N]" -n 1 -r; echo if [[ ! $REPLY =~ ^[Yy]$ ]]; then exit 1; fi fi ${debug} echo -n "Creating VBD: " >&2 - vbd_uuid=$(${XE} vbd-create vm-uuid=${CONTROL_DOMAIN_UUID} vdi-uuid=${vdi_uuid} device=autodetect 2>/dev/null) + vbd_uuid=$(${XE} vbd-create vm-uuid="${CONTROL_DOMAIN_UUID}" vdi-uuid="${vdi_uuid}" device=autodetect 2>/dev/null) if [ $? -ne 0 -o -z "${vbd_uuid}" ]; then - ${debug} echo error creating VBD for VDI ${vdi_uuid} >&2 + ${debug} echo "error creating VBD for VDI ${vdi_uuid}" >&2 cleanup continue fi - ${debug} echo ${vbd_uuid} >&2 + ${debug} echo "${vbd_uuid}" >&2 ${debug} echo -n "Plugging VBD: " >&2 - ${XE} vbd-plug uuid=${vbd_uuid} - device=/dev/$(${XE} vbd-param-get uuid=${vbd_uuid} param-name=device) + ${XE} vbd-plug uuid="${vbd_uuid}" + device=/dev/$(${XE} vbd-param-get uuid="${vbd_uuid}" param-name=device) - if [ ! -b ${device} ]; then - ${debug} echo ${device}: not a block special >&2 + if [ ! -b "${device}" ]; then + ${debug} echo "${device}: not a block special" >&2 cleanup continue fi - ${debug} echo ${device} >&2 + ${debug} echo "${device}" >&2 ${debug} echo -n "Probing device: " >&2 probecmd="@LIBEXECDIR@/probe-device-for-file" metadata_stamp="/.ctxs-metadata-backup" mnt= - ${probecmd} ${device} ${metadata_stamp} + ${probecmd} "${device}" "${metadata_stamp}" if [ $? -eq 0 ]; then - ${debug} echo found metadata backup >&2 + ${debug} echo "found metadata backup" >&2 ${debug} echo -n "Mounting filesystem: " >&2 - mnt=/var/run/pool-backup-${vdi_uuid} - mkdir -p ${mnt} - /sbin/fsck -a ${device} >/dev/null 2>&1 + mnt="/var/run/pool-backup-${vdi_uuid}" + mkdir -p "${mnt}" + /sbin/fsck -a "${device}" >/dev/null 2>&1 if [ $? -ne 0 ]; then - echo File system integrity error. Please correct manually. >&2 + echo "File system integrity error. Please correct manually." >&2 cleanup continue fi - mount ${device} ${mnt} >/dev/null 2>&1 + mount "${device}" "${mnt}" >/dev/null 2>&1 if [ $? -ne 0 ]; then ${debug} echo failed >&2 cleanup else if [ -e "${mnt}/.ctxs-metadata-backup" ]; then - ${debug} echo Found backup metadata on VDI: ${vdi_uuid} >&2 - xe vdi-param-set uuid=${vdi_uuid} other-config:ctxs-pool-backup=true + ${debug} echo "Found backup metadata on VDI: ${vdi_uuid}" >&2 + xe vdi-param-set uuid="${vdi_uuid}" other-config:ctxs-pool-backup=true break fi fi else - ${debug} echo backup metadata not found >&2 + ${debug} echo "backup metadata not found" >&2 fi cleanup done if [ $just_probe -gt 0 ]; then - echo ${vdi_uuid} + echo "${vdi_uuid}" cleanup exit 0 fi -cd ${mnt} +cd "${mnt}" ${debug} echo "" >&2 -if [ ! -d ${mnt}/metadata ]; then - echo Metadata backups not found. >&2 +if [ ! -d "${mnt}/metadata" ]; then + echo "Metadata backups not found." >&2 cleanup exit 1 fi -cd ${mnt}/metadata +cd "${mnt}/metadata" -if [ $just_list_dates -gt 0 ]; then - ls -1r ${mnt}/metadata +if [ "$just_list_dates" -gt 0 ]; then + ls -1r "${mnt}/metadata" cleanup exit 0 fi @@ -253,39 +253,39 @@ fi if [ -z "${chosen_date}" ]; then chosen_metadata_dir=$(ls | sort -n | tail -1) if [ -z "${chosen_metadata_dir}" ]; then - echo No VM metadata backups found in ${mnt}/metadata >&2 + echo "No VM metadata backups found in ${mnt}/metadata" >&2 cleanup exit 1 fi else - if [ ! -d ${mnt}/metadata/${chosen_date} ]; then - echo Date directory "${chosen_date}" not found >&2 + if [ ! -d "${mnt}/metadata/${chosen_date}" ]; then + echo "Date directory ${chosen_date} not found" >&2 cleanup exit 1 fi - chosen_metadata_dir=${chosen_date} + chosen_metadata_dir="${chosen_date}" fi case ${restore_mode} in sr) - full_dir=${mnt}/metadata/${chosen_metadata_dir}/by-sr/${sr_uuid} + full_dir="${mnt}/metadata/${chosen_metadata_dir}/by-sr/${sr_uuid}" ;; all) - full_dir=${mnt}/metadata/${chosen_metadata_dir}/all + full_dir="${mnt}/metadata/${chosen_metadata_dir}/all" ;; esac if [ ! -d "${full_dir}" ]; then - echo No VM metadata exports were found for the selected SR >&2 + echo "No VM metadata exports were found for the selected SR" >&2 cleanup exit 1 fi -${debug} echo Selected: ${full_dir} +${debug} echo "Selected: ${full_dir}" -cd ${full_dir} +cd "${full_dir}" ${debug} echo "" >&2 -${debug} echo Latest VM metadata found is: >&2 +${debug} echo "Latest VM metadata found is": >&2 ${debug} ls >&2 if [ "$yes" -eq 0 ]; then @@ -293,14 +293,14 @@ if [ "$yes" -eq 0 ]; then echo "Please type in 'yes' and to continue." read response if [ "$response" != "yes" ]; then - echo Aborting metadata restore. + echo "Aborting metadata restore." cleanup exit 1 fi fi ${debug} echo "" >&2 -${debug} echo Restoring VM metadata: >&2 +${debug} echo "Restoring VM metadata:" >&2 trap - ERR From 7f1d315135651a84d39f0512fc433f28f3bdba33 Mon Sep 17 00:00:00 2001 From: Christian Lindig Date: Thu, 21 Mar 2024 14:38:59 +0000 Subject: [PATCH 050/149] CP-46179 use -y (yes) flag to facilitate scripting We have added an interactive prompt in case the backup VDI was created by an earlier version and does not match the expected UUID. To facilitate scripting, add a -y (yes) flag that assumes a confirmation. Signed-off-by: Christian Lindig --- scripts/xe-backup-metadata | 7 +++++-- scripts/xe-restore-metadata | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/xe-backup-metadata b/scripts/xe-backup-metadata index d6dc9b877f9..47b21108b9d 100755 --- a/scripts/xe-backup-metadata +++ b/scripts/xe-backup-metadata @@ -39,6 +39,7 @@ function usage { echo " -k: Number of older backups to preserve (default: ${history_kept})" echo " -n: Just try to find a backup VDI and stop the script after that" echo " -f Force backup even when less than 10% free capacity is left on the backup VDI" + echo " -y: Assume non-interactive mode and yes to all questions" echo " -v: Verbose output" echo echo @@ -86,7 +87,8 @@ just_find_vdi=0 fs_uninitialised=0 usage_alert=90 force_backup=0 -while getopts "hvink:u:dcf" opt ; do +yes=0 +while getopts "yhvink:u:dcf" opt ; do case $opt in h) usage ;; c) create_vdi=1 ; fs_uninitialised=1 ;; @@ -96,6 +98,7 @@ while getopts "hvink:u:dcf" opt ; do d) leave_mounted=1 ;; n) just_find_vdi=1 ;; v) debug="" ;; + y) yes=1 ;; f) force_backup=1 ;; *) echo "Invalid option"; usage ;; esac @@ -159,7 +162,7 @@ function cleanup { # if we can't validate the UUID of the VDI, prompt the user if [ -n "${vdi_uuid}" ]; then - if ! validate_vdi_uuid "${sr_uuid}" "${vdi_uuid}"; then + if ! validate_vdi_uuid "${sr_uuid}" "${vdi_uuid}" && [ "$yes" -eq 0 ]; then echo "Backup VDI $vdi_uuid was most likley create by an earlier" echo "version of this code. Make sure this is a VDI that you" echo "created as we can't validate it without mounting it." diff --git a/scripts/xe-restore-metadata b/scripts/xe-restore-metadata index d9ff3d2d37e..093cd772192 100755 --- a/scripts/xe-restore-metadata +++ b/scripts/xe-restore-metadata @@ -163,7 +163,7 @@ fi trap cleanup SIGINT ERR for vdi_uuid in ${vdis}; do - if [ "${vdi_uuid}" != "${backup_vdi}" ]; then + if [ "${vdi_uuid}" != "${backup_vdi}" ] && [ "$yes" -eq 0 ]; then echo "Probing VDI ${vdi_uuid}." echo "This VDI was created with a prior version of this code." echo "Its validity can't be checked without mounting it first." From d37b86684ca9a982884a2b681ee8616501a54005 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 6 Mar 2024 16:30:03 +0000 Subject: [PATCH 051/149] CA-388860: set xapi-guard cache capacity to 512 This is about the same order of magnitude as the number of VMs supported in a host Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index 0f0a6e2c248..11fb40b5aaf 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -595,7 +595,8 @@ end let setup typ read write = let* () = Setup.retime_cache_contents typ in - let queue, push = Lwt_bounded_stream.create 2 in + let capacity = 512 in + let queue, push = Lwt_bounded_stream.create capacity in let lock = Lwt_mutex.create () in let q = {queue; push; lock; state= Disengaged} in Lwt.return From ec68831036a0f450a16759e82ea2eedbb49b967b Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Thu, 7 Mar 2024 17:40:45 +0000 Subject: [PATCH 052/149] xapi-guard-test: Direct loglines to stdout This helps when running it to observe behaviour of the library Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/test/cache_test.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/ocaml/xapi-guard/test/cache_test.ml b/ocaml/xapi-guard/test/cache_test.ml index 97b144839a6..9812091f0a7 100644 --- a/ocaml/xapi-guard/test/cache_test.ml +++ b/ocaml/xapi-guard/test/cache_test.ml @@ -200,5 +200,6 @@ let main () = Lwt.return_unit let () = + Debug.log_to_stdout () ; setup_log @@ Some Logs.Debug ; Lwt_main.run (main ()) From 64fdc740ee80533baabaef59a45fa5aba1c35625 Mon Sep 17 00:00:00 2001 From: Gabriel Buica Date: Tue, 26 Mar 2024 09:10:35 +0000 Subject: [PATCH 053/149] CP-46576: Add standard http attributes Add standard http attributes to from opentelemetry such as: - `http.request.method` - `http.request.header.content-type`; - `http.request.body.size`; - `http.request.header.user-agent`; - `http.request.header.(key)`. This improves debuggability associated with client requests. Signed-off-by: Gabriel Buica --- ocaml/xapi/context.ml | 34 +++++++++++++++++++++++++++++++--- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/ocaml/xapi/context.ml b/ocaml/xapi/context.ml index 4179cf7d930..fc611de7fb5 100644 --- a/ocaml/xapi/context.ml +++ b/ocaml/xapi/context.ml @@ -227,8 +227,36 @@ let parent_of_origin (origin : origin) span_name = | _ -> None +let attribute_helper_fn f v = Option.fold ~none:[] ~some:f v + +let attr_of_req (req : Http.Request.t) = + [ + [ + ("xs.xapi.task.origin", "http") + ; ("http.request.header.method", Http.string_of_method_t req.m) + ] + ; attribute_helper_fn + (fun user_agent -> [("http.request.header.user-agent", user_agent)]) + req.user_agent + ; attribute_helper_fn + (fun content_type -> [("http.request.header.content-type", content_type)]) + req.content_type + ; attribute_helper_fn + (fun content_length -> + [("http.request.body.size", Printf.sprintf "%Li" content_length)] + ) + req.content_length + ; List.map + (fun (h, v) -> + ( h |> String.lowercase_ascii |> Printf.sprintf "http.request.header.%s" + , v + ) + ) + req.additional_headers + ] + |> List.concat + let make_attributes ?task_name ?task_id ?task_uuid ?session_id ?origin () = - let attribute_helper_fn f v = Option.fold ~none:[] ~some:f v in [ attribute_helper_fn (fun task_name -> [("xs.xapi.task.name", task_name)]) @@ -249,8 +277,8 @@ let make_attributes ?task_name ?task_id ?task_uuid ?session_id ?origin () = match origin with | Internal -> [("xs.xapi.task.origin", "internal")] - | Http _ -> - [("xs.xapi.task.origin", "http")] + | Http (req, _) -> + attr_of_req req ) origin ] From 364073a2ea476ebf2acec57cd24cc6b841afd273 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 6 Mar 2024 17:13:44 +0000 Subject: [PATCH 054/149] CA-388860: xapi-guard cache now starts in the most efficient mode This reuses the re-key process to gather current contents of the cache and depending on the amount of elements pending it 1. Bypasses the cache completely if it's empty 2. Uses the queue if all the elements fit in the queue 3. Starts as before if there are more elements Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 40 ++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index 11fb40b5aaf..0a35d9ba7fb 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -519,25 +519,28 @@ end (** Module use to change the cache contents before the reader and writer start running *) module Setup : sig - val retime_cache_contents : Types.Service.t -> unit Lwt.t + val retime_cache_contents : Types.Service.t -> t List.t Lwt.t + (** [retime_cache_contents typ] retimes the current cache contents so they + are time congruently with the current execution and returns the keys of + valid files that are yet to be pushed *) end = struct type file_action = | Keep of file | Delete of string | Move of {from: string; into: string} - let get_fs_action root now = function + let get_fs_action root now acc = function | Latest ((uuid, timestamp, key), from) as latest -> if Mtime.is_later ~than:now timestamp then let timestamp = now in let into = path_of_key root (uuid, timestamp, key) in - Move {from; into} + ((uuid, timestamp, key) :: acc, Move {from; into}) else - Keep latest + ((uuid, timestamp, key) :: acc, Keep latest) | Temporary _ as temp -> - Keep temp + (acc, Keep temp) | Invalid p | Outdated (_, p) -> - Delete p + (acc, Delete p) let commit __FUN = function | Keep (Temporary p) -> @@ -585,20 +588,31 @@ end = struct let now = Mtime_clock.now () in let root = cache_of typ in let* contents = get_all_contents root in - let* () = - contents - |> List.map (get_fs_action root now) - |> Lwt_list.iter_p (commit __FUNCTION__) + let pending, actions = + contents |> List.fold_left_map (get_fs_action root now) [] in - delete_empty_dirs ~delete_root:false root + let* () = Lwt_list.iter_p (commit __FUNCTION__) actions in + let* () = delete_empty_dirs ~delete_root:false root in + Lwt.return pending end let setup typ read write = - let* () = Setup.retime_cache_contents typ in + let* pending = Setup.retime_cache_contents typ in let capacity = 512 in let queue, push = Lwt_bounded_stream.create capacity in let lock = Lwt_mutex.create () in - let q = {queue; push; lock; state= Disengaged} in + let state = + if pending = [] then + Direct + else if List.length pending < capacity then + let () = + List.iter (fun e -> Option.value ~default:() (push (Some e))) pending + in + Engaged + else + Disengaged + in + let q = {queue; push; lock; state} in Lwt.return ( Writer.with_cache ~direct:(read, write) typ q , Watcher.watch ~direct:write typ q From b06b27c42ee314da16156bf76ac08a179bb2a6b4 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 8 Mar 2024 09:46:39 +0000 Subject: [PATCH 055/149] CA-388860: xapi-guard cache: protect against invalid keys Unknown directories detected in the cache will get logged, and at the appropriate times deleted, along with all the files contained within. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 7 ++++++- ocaml/xapi-guard/lib/types.ml | 8 ++++---- ocaml/xapi-guard/lib/types.mli | 2 +- ocaml/xapi-guard/test/cache_test.ml | 4 ++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index 0a35d9ba7fb..ee91bf4f396 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -81,7 +81,12 @@ let key_of_path path = let* key = Filename.basename key_dir |> int_of_string_opt - |> Option.map Types.Tpm.deserialize_key + |> Option.map (fun e -> + Types.Tpm.deserialize_key e + |> Result.map_error (fun msg -> D.info "Invalid key found: %s" msg) + |> Result.to_option + ) + |> Option.join in let* timestamp = Filename.basename path diff --git a/ocaml/xapi-guard/lib/types.ml b/ocaml/xapi-guard/lib/types.ml index 3f2b41c7682..ff6dbc1dd3c 100644 --- a/ocaml/xapi-guard/lib/types.ml +++ b/ocaml/xapi-guard/lib/types.ml @@ -28,13 +28,13 @@ module Tpm = struct let deserialize_key = function | 0 -> - Perm + Ok Perm | 1 -> - Save + Ok Save | 2 -> - Volatile + Ok Volatile | s -> - Fmt.invalid_arg "Unknown TPM state key: %i" s + Error Printf.(sprintf "Unknown TPM state key: %i" s) let empty_state = "" diff --git a/ocaml/xapi-guard/lib/types.mli b/ocaml/xapi-guard/lib/types.mli index f210ea8c96a..06b811ba30c 100644 --- a/ocaml/xapi-guard/lib/types.mli +++ b/ocaml/xapi-guard/lib/types.mli @@ -17,7 +17,7 @@ module Tpm : sig (** [key_of_swtpm path] returns a state key represented by [path]. These paths are parts of the requests generated by SWTPM and may contain slashes *) - val deserialize_key : int -> key + val deserialize_key : int -> (key, string) Result.t val serialize_key : key -> int (** [serialize key] returns the state key represented by [key]. *) diff --git a/ocaml/xapi-guard/test/cache_test.ml b/ocaml/xapi-guard/test/cache_test.ml index 9812091f0a7..3e51cab2c35 100644 --- a/ocaml/xapi-guard/test/cache_test.ml +++ b/ocaml/xapi-guard/test/cache_test.ml @@ -12,7 +12,7 @@ module TPMs = struct let request_persist uuid write = let __FUN = __FUNCTION__ in - let key = Tpm.deserialize_key (Random.int 3) in + let key = Tpm.deserialize_key (Random.int 3) |> Result.get_ok in let time = Mtime_clock.now () in let serial_n = Atomic.fetch_and_add writes_created 1 in @@ -31,7 +31,7 @@ module TPMs = struct let request_read uuid read = let __FUN = __FUNCTION__ in - let key = Tpm.deserialize_key (Random.int 3) in + let key = Tpm.deserialize_key (Random.int 3) |> Result.get_ok in let time = Mtime_clock.now () in let serial_n = Atomic.fetch_and_add reads_created 1 in From 7f0afec7e5063f755881916f98abe90552b1fdfd Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 11 Mar 2024 15:15:22 +0000 Subject: [PATCH 056/149] CP-48385: xapi-guard cache: do not spam logs on errors Previously a logline was created per push attempt, now up to two per failure period are done: one on the first failure, and one on recovery. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 30 +++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index ee91bf4f396..cc50b5c6520 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -68,12 +68,13 @@ type file = | Temporary of string | Invalid of string -let path_of_key root (uuid, timestamp, key) = - root - // Uuidm.to_string uuid +let print_key (uuid, timestamp, key) = + Uuidm.to_string uuid // Types.Tpm.(serialize_key key |> string_of_int) // Mtime.(to_uint64_ns timestamp |> Int64.to_string) +let path_of_key root key = root // print_key key + let key_of_path path = let ( let* ) = Option.bind in let key_dir = Filename.(dirname path) in @@ -412,18 +413,29 @@ end = struct let retry_push push (uuid, timestamp, key) contents = let __FUN = __FUNCTION__ in let push' () = push (uuid, timestamp, key) contents in - let rec retry k = + let counter = Mtime_clock.counter () in + let rec retry is_first_try = let on_error e = - D.info "%s: Error on push, attempt %i. Reason: %s" __FUN k - (Printexc.to_string e) ; + if is_first_try then + D.debug "%s: Error on push, retrying. Reason: %s" __FUN + (Printexc.to_string e) ; let* () = Lwt_unix.sleep 0.1 in - retry (k + 1) + retry false in Lwt.try_bind push' - (function Ok () -> Lwt.return_unit | Error e -> on_error e) + (function + | Ok () -> Lwt.return (not is_first_try) | Error e -> on_error e + ) on_error in - retry 1 + let* failed = retry true in + ( if failed then + let elapsed = Mtime_clock.count counter in + D.debug "%s: Pushed %s after trying for %s" __FUN + (print_key (uuid, timestamp, key)) + (Fmt.to_to_string Mtime.Span.pp elapsed) + ) ; + Lwt.return_unit let push_file push (key, path) = let __FUN = __FUNCTION__ in From 7268e368ad07f3bfff20192697838447213a0753 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 11 Mar 2024 15:44:44 +0000 Subject: [PATCH 057/149] CP-48385: xapi-guard: Remove the notion of temporary writes Now temporary files are an internal detail of the persistence function. The rest of code needs not be aware of it: if some of temp files linger in the filesystem means that the process did not acknowledge a requested write, so the domain could not have possibly acted on the write. Delete these as this does not incur in any data corruption in the domain side. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-guard/lib/disk_cache.ml | 58 ++++++------------------------ 1 file changed, 10 insertions(+), 48 deletions(-) diff --git a/ocaml/xapi-guard/lib/disk_cache.ml b/ocaml/xapi-guard/lib/disk_cache.ml index cc50b5c6520..5e8b9bb0650 100644 --- a/ocaml/xapi-guard/lib/disk_cache.ml +++ b/ocaml/xapi-guard/lib/disk_cache.ml @@ -62,11 +62,7 @@ let unlink_safe file = type valid_file = t * string -type file = - | Latest of valid_file - | Outdated of valid_file - | Temporary of string - | Invalid of string +type file = Latest of valid_file | Outdated of valid_file | Invalid of string let print_key (uuid, timestamp, key) = Uuidm.to_string uuid @@ -96,24 +92,17 @@ let key_of_path path = in Some ((uuid, timestamp, key), path) -let path_is_temp path = - let pathlen = String.length path in - String.ends_with ~suffix:".pre" path - && key_of_path (String.sub path 0 (pathlen - 4)) |> Option.is_some - -let temp_of_path path = path ^ ".pre" +let only_latest = function + | Latest f -> + Either.Left f + | Outdated (_, p) | Invalid p -> + Right p let sort_updates contents = let classify elem = match key_of_path elem with | None -> - let file = - if path_is_temp elem then - Temporary elem - else - Invalid elem - in - Either.Right file + Either.Right (Invalid elem) | Some valid_file -> Either.Left valid_file in @@ -158,7 +147,7 @@ let read_from ~filename = let persist_to ~filename:f_path ~contents = let atomic_write_to_file ~perm f = - let tmp_path = temp_of_path f_path in + let tmp_path = f_path ^ ".pre" in let dirname = Filename.dirname f_path in let flags = Unix.[O_WRONLY; O_CREAT; O_SYNC] in let* fd_tmp = Lwt_unix.openfile tmp_path flags perm in @@ -291,16 +280,10 @@ end = struct let updates = sort_updates contents in (* 2. Pick latest *) - let only_latest = function - | Latest (_, p) -> - Either.Left p - | Temporary p | Outdated (_, p) | Invalid p -> - Right p - in let latest, _ = List.partition_map only_latest updates in (* 3. fall back to remote read if needed *) - let get_contents path = + let get_contents (_, path) = Lwt.catch (fun () -> read_from ~filename:path) (fun _ -> read_remote ()) in @@ -388,26 +371,10 @@ module Watcher : sig end = struct type push_cache = File of valid_file | Update_all | Wait - (* Outdated and invalid files can be deleted, keep temporary files just in case - they need to be recovered *) - let discarder = function - | Latest _ as f -> - Either.Left f - | Temporary _ as f -> - Left f - | Outdated (_, p) -> - Right p - | Invalid p -> - Right p - let get_latest_and_delete_rest root = let* files = get_all_contents root in - let keep, to_delete = List.partition_map discarder files in + let latest, to_delete = List.partition_map only_latest files in let* () = Lwt_list.iter_p unlink_safe to_delete in - (* Ignore temporaty files *) - let latest = - List.filter_map (function Latest f -> Some f | _ -> None) keep - in Lwt.return latest let retry_push push (uuid, timestamp, key) contents = @@ -554,15 +521,10 @@ end = struct ((uuid, timestamp, key) :: acc, Move {from; into}) else ((uuid, timestamp, key) :: acc, Keep latest) - | Temporary _ as temp -> - (acc, Keep temp) | Invalid p | Outdated (_, p) -> (acc, Delete p) let commit __FUN = function - | Keep (Temporary p) -> - D.warn "%s: Found temporary file, ignoring '%s'" __FUN p ; - Lwt.return_unit | Keep _ -> Lwt.return_unit | Delete p -> From 66f33ace5bf7d6f000c0d7ec64062f37339914fe Mon Sep 17 00:00:00 2001 From: Ming Lu Date: Thu, 28 Mar 2024 14:16:06 +0800 Subject: [PATCH 058/149] CA-384483: Can't export VDI to VHD file with base VDI With *hybrid source format in export, the following cases are supported: 1. nbdhybrid: QCOW2 -> NBD device in dom0 -> exported file 2. nbdhybrid: VHD -> NBD device in dom0 -> exported file 3. hybrid: VHD -> blktap device in dom0 -> exported file The case 2 above can't support an optional parameter "base". This parameter holds the ID of another VHD VDI. When it is passed, the export will only write the differences between "source" and "base" to the destination file. As a short-term solution, in case 2 above, when the "base" is passed, the source format is changed to "hybrid" in this commit. This can work because: 1. the comparsion on blocks required by "base" is supported by "hybrid"; 2. the raw data from NBD device and blktap (Frankentap) device are same; 3. the sparseness information of the source required in case 2 can be get by either NBD interface (nbdhybrid) or VHD parsing (hybrid). Signed-off-by: Ming Lu --- ocaml/xapi/stream_vdi.ml | 25 +++++++++++++++++++++++++ ocaml/xapi/vhd_tool_wrapper.ml | 22 ++++++++++++++++------ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/ocaml/xapi/stream_vdi.ml b/ocaml/xapi/stream_vdi.ml index 64b1da93eee..3c27d158af3 100644 --- a/ocaml/xapi/stream_vdi.ml +++ b/ocaml/xapi/stream_vdi.ml @@ -163,6 +163,31 @@ let get_nbd_device path = else None +(* Copied from vhd-tool/src/image.ml. + * Just keep the situation of xapi doesn't depend on vhd-tool OCaml module. + *) +let image_behind_nbd_device = function + | Some (path, _exportname) as image -> + (* The nbd server path exposed by tapdisk can lead us to the actual image + file below. Following the symlink gives a path like + `/run/blktap-control/nbd.`, + containing the tapdisk pid and minor number. Using this information, + we can get the file path from tap-ctl. + *) + let default _ _ = image in + let filename = Unix.realpath path |> Filename.basename in + Scanf.ksscanf filename default "nbd%d.%d" (fun pid minor -> + match Tapctl.find (Tapctl.create ()) ~pid ~minor with + | _, _, Some ("vhd", vhd) -> + Some ("vhd", vhd) + | _, _, Some ("aio", vhd) -> + Some ("raw", vhd) + | _, _, _ | (exception _) -> + None + ) + | _ -> + None + type extent = {flags: int32; length: int64} [@@deriving rpc] type extent_list = extent list [@@deriving rpc] diff --git a/ocaml/xapi/vhd_tool_wrapper.ml b/ocaml/xapi/vhd_tool_wrapper.ml index 7e22dc86597..6fe4e40d70d 100644 --- a/ocaml/xapi/vhd_tool_wrapper.ml +++ b/ocaml/xapi/vhd_tool_wrapper.ml @@ -170,9 +170,16 @@ let vhd_of_device path = | _, _, _ -> raise Not_found with - | Tapctl.Not_blktap -> + | Tapctl.Not_blktap -> ( debug "Device %s is not controlled by blktap" path ; - None + (* Check if it is a VHD behind a NBD deivce *) + Stream_vdi.(get_nbd_device path |> image_behind_nbd_device) |> function + | Some ("vhd", vhd) -> + debug "%s is a VHD behind NBD device %s" vhd path ; + Some vhd + | _ -> + None + ) | Tapctl.Not_a_device -> debug "%s is not a device" path ; None @@ -186,15 +193,18 @@ let send progress_cb ?relative_to (protocol : string) (dest_format : string) (s : Unix.file_descr) (path : string) (size : Int64.t) (prefix : string) = let s' = Uuidx.(to_string (make ())) in let source_format, source = - match (Stream_vdi.get_nbd_device path, vhd_of_device path) with - | Some (nbd_server, exportname), _ -> + match (Stream_vdi.get_nbd_device path, vhd_of_device path, relative_to) with + | Some (nbd_server, exportname), _, None -> ( "nbdhybrid" , Printf.sprintf "%s:%s:%s:%Ld" path nbd_server exportname size ) - | None, Some vhd -> + | Some _, Some vhd, Some _ | None, Some vhd, _ -> ("hybrid", path ^ ":" ^ vhd) - | None, None -> + | None, None, None -> ("raw", path) + | _, None, Some _ -> + let msg = "Cannot compute differences on non-VHD images" in + error "%s" msg ; failwith msg in let relative_to = match relative_to with From 8017465b71f8f15b610fb902c5a843087e3c1c04 Mon Sep 17 00:00:00 2001 From: Christian Lindig Date: Fri, 23 Jun 2023 14:18:41 +0100 Subject: [PATCH 059/149] CA-378317 fix EBADF in waitpid_nohang (v3) waitpid_nohang closes the socket by calling waitpid and then calls clear_nonblock on that socket, which fails. waitpid_nohang's policy around closing the socket is not obvious enough. Make it more explicit by not calling waitpid but inlining code and looking at the different cases. The behaviour is still to close the socket when the child process has terminated and not otherwise. Logging can be re-enabled if we have to come back to this. Signed-off-by: Christian Lindig --- ocaml/forkexecd/lib/forkhelpers.ml | 49 ++++++++++++++++++++++++----- ocaml/forkexecd/lib/forkhelpers.mli | 6 ++-- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/ocaml/forkexecd/lib/forkhelpers.ml b/ocaml/forkexecd/lib/forkhelpers.ml index d55901c3c68..15fb4bca6c5 100644 --- a/ocaml/forkexecd/lib/forkhelpers.ml +++ b/ocaml/forkexecd/lib/forkhelpers.ml @@ -21,6 +21,8 @@ (* XXX: this is a work in progress *) +module D = Debug.Make (struct let name = __MODULE__ end) + let default_path = ["/sbin"; "/usr/sbin"; "/bin"; "/usr/bin"] let default_path_env_pair = [|"PATH=" ^ String.concat ":" default_path|] @@ -72,14 +74,47 @@ let waitpid (sock, pid) = in failwith msg -let waitpid_nohang ((sock, _) as x) = +(* [waitpid_nohang] reports the status of a socket to a process. The + intention is to make this non-blocking. If the process is finished, + the socket is closed and not otherwise. *) +let waitpid_nohang (sock, pid) = + let verbose = false in + if verbose then D.debug "%s pid=%d" __FUNCTION__ pid ; + let fail fmt = Printf.kprintf failwith fmt in Unix.set_nonblock sock ; - let r = - try waitpid x - with Unix.(Unix_error ((EAGAIN | EWOULDBLOCK), _, _)) -> - (0, Unix.WEXITED 0) - in - Unix.clear_nonblock sock ; r + match Fecomms.read_raw_rpc sock with + | Ok Fe.(Finished (WEXITED n)) -> + if verbose then D.debug "%s pid=%d WEXITED" __FUNCTION__ pid ; + Unix.close sock ; + (pid, Unix.WEXITED n) + | Ok Fe.(Finished (WSIGNALED n)) -> + if verbose then D.debug "%s pid=%d WSIGNALED" __FUNCTION__ pid ; + Unix.close sock ; + (pid, Unix.WSIGNALED n) + | Ok Fe.(Finished (WSTOPPED n)) -> + if verbose then D.debug "%s pid=%d WSTOPPED" __FUNCTION__ pid ; + Unix.close sock ; + (pid, Unix.WSTOPPED n) + | Ok status -> + Unix.clear_nonblock sock ; + fail "%s: unexpected status received (%s)" __FUNCTION__ + (Fe.ferpc_to_string status) + | Error msg -> + D.debug "%s pid=%d %s" __FUNCTION__ pid msg ; + Unix.clear_nonblock sock ; + fail "%s: error happened when trying to read the status. %s" __FUNCTION__ + msg + (* it's a bit crazy that we have Result.t and exceptions from + read_raw_rpc *) + | exception Unix.(Unix_error ((EAGAIN | EWOULDBLOCK), _, _)) -> + if verbose then D.debug "%s pid=%d EAGAIN EWOULDBLOCK" __FUNCTION__ pid ; + Unix.clear_nonblock sock ; + (0, Unix.WEXITED 0) (* this a convention, see MLI *) + | exception exn -> + D.debug "%s pid=%d %s" __FUNCTION__ pid (Printexc.to_string exn) ; + Unix.clear_nonblock sock ; + fail "%s: error happened when trying to read the status. %s" __FUNCTION__ + (Printexc.to_string exn) let dontwaitpid (sock, _pid) = ( try diff --git a/ocaml/forkexecd/lib/forkhelpers.mli b/ocaml/forkexecd/lib/forkhelpers.mli index 6252f0e75ca..48832726c02 100644 --- a/ocaml/forkexecd/lib/forkhelpers.mli +++ b/ocaml/forkexecd/lib/forkhelpers.mli @@ -111,8 +111,10 @@ val waitpid : pidty -> int * Unix.process_status (** [waitpid p] returns the (pid, Unix.process_status) *) val waitpid_nohang : pidty -> int * Unix.process_status -(** [waitpid_nohang p] returns the (pid, Unix.process_status) if the process has already - quit or (0, Unix.WEXITTED 0) if the process is still running. *) +(** [waitpid_nohang p] returns the (pid, Unix.process_status) if the + process has already quit or (0, Unix.WEXITTED 0) if the process is + still running. If the process is finished, the socket is closed + and not otherwise. *) val dontwaitpid : pidty -> unit (** [dontwaitpid p]: signals the caller's desire to never call waitpid. Note that the final From 599a9c6235c1485951621f6ddfdff855484278ee Mon Sep 17 00:00:00 2001 From: Lunfan Zhang Date: Fri, 29 Mar 2024 03:25:34 +0000 Subject: [PATCH 060/149] CP-47660 define anti-affinity feature Signed-off-by: Lunfan Zhang --- ocaml/xapi-types/features.ml | 4 ++++ ocaml/xapi-types/features.mli | 1 + 2 files changed, 5 insertions(+) diff --git a/ocaml/xapi-types/features.ml b/ocaml/xapi-types/features.ml index 37fafc0905a..d55d7d01c37 100644 --- a/ocaml/xapi-types/features.ml +++ b/ocaml/xapi-types/features.ml @@ -64,6 +64,7 @@ type feature = | Updates | Internal_repo_access | VTPM + | VM_anti_affinity [@@deriving rpc] type orientation = Positive | Negative @@ -132,6 +133,9 @@ let keys_of_features = , ("restrict_internal_repo_access", Negative, "Internal_repo_access") ) ; (VTPM, ("restrict_vtpm", Negative, "VTPM")) + ; ( VM_anti_affinity + , ("restrict_vm_anti_affinity", Negative, "VM_anti_affinity") + ) ] (* A list of features that must be considered "enabled" by `of_assoc_list` diff --git a/ocaml/xapi-types/features.mli b/ocaml/xapi-types/features.mli index c2f1ed2a51b..0696b3ddb5e 100644 --- a/ocaml/xapi-types/features.mli +++ b/ocaml/xapi-types/features.mli @@ -72,6 +72,7 @@ type feature = | Internal_repo_access (** Enable restriction on repository access to pool members only *) | VTPM (** Support VTPM device required by Win11 guests *) + | VM_anti_affinity (** Enable use of VM anti-affinity placement *) val feature_of_rpc : Rpc.t -> feature (** Convert RPC into {!feature}s *) From ec36f8b3bcb432398caa357522511ea9f8a58be0 Mon Sep 17 00:00:00 2001 From: Benjamin Reis Date: Fri, 29 Mar 2024 16:53:29 +0100 Subject: [PATCH 061/149] Use Magic MIME to determine fileserver's response content type Signed-off-by: Benjamin Reis --- ocaml/xapi/dune | 1 + ocaml/xapi/fileserver.ml | 34 +--------------------------------- 2 files changed, 2 insertions(+), 33 deletions(-) diff --git a/ocaml/xapi/dune b/ocaml/xapi/dune index 45d5e67aaf9..fe161e0dd5f 100644 --- a/ocaml/xapi/dune +++ b/ocaml/xapi/dune @@ -73,6 +73,7 @@ hex http_lib ipaddr + magic-mime message-switch-core message-switch-unix mirage-crypto diff --git a/ocaml/xapi/fileserver.ml b/ocaml/xapi/fileserver.ml index 1c4cf9520e3..ed9ed334d66 100644 --- a/ocaml/xapi/fileserver.ml +++ b/ocaml/xapi/fileserver.ml @@ -42,40 +42,8 @@ let missing uri = ^ " was not found on this server.

Xapi \ Server
" -let get_extension filename = - try - let basename = Filename.basename filename in - let i = String.rindex basename '.' in - Some (String.sub basename (i + 1) (String.length basename - i - 1)) - with _ -> None - -let application_octet_stream = "application/octet-stream" - -let mime_of_extension = function - | "html" | "htm" -> - "text/html" - | "css" -> - "text/css" - | "js" -> - "application/javascript" - | "gif" -> - "image/gif" - | "png" -> - "image/png" - | "jpg" | "jpeg" -> - "image/jpeg" - | "xml" -> - "application/xml" - | "rpm" -> - "application/x-rpm" - | _ -> - application_octet_stream - let response_file s file_path = - let mime_content_type = - let ext = Option.map String.lowercase_ascii (get_extension file_path) in - Option.fold ~none:application_octet_stream ~some:mime_of_extension ext - in + let mime_content_type = Magic_mime.lookup file_path in let hsts_time = !Xapi_globs.hsts_max_age in Http_svr.response_file ~mime_content_type ~hsts_time s file_path From dcbdbb98b12203efd81cba5f1eeae03c09e4c1ca Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Sun, 17 Mar 2024 12:21:29 +0000 Subject: [PATCH 062/149] Add minor test to forkexec test_exitcode test Slightly different from missing file, add missing path. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/test/fe_test.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/ocaml/forkexecd/test/fe_test.ml b/ocaml/forkexecd/test/fe_test.ml index bb740d94df8..e6c4ae21017 100644 --- a/ocaml/forkexecd/test/fe_test.ml +++ b/ocaml/forkexecd/test/fe_test.ml @@ -150,6 +150,7 @@ let test_exitcode () = in run_expect "/bin/false" 1 ; run_expect "/bin/xe-fe-test-no-command" 127 ; + run_expect "/bin/xe-fe-no-path/xe-fe-test-no-command" 127 ; run_expect "/etc/hosts" 126 ; Printf.printf "\nCompleted exitcode tests\n" From 28135b59b061b7efc4ba24046b3944467fb8202a Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Thu, 4 Apr 2024 14:13:58 +0000 Subject: [PATCH 063/149] Make it easier to run forkexecd tests manually Clean even if user press Ctrl-C, not only sending TERM signal. Remove possible stale unix socket to allow to run multiple times. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/test/fe_test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ocaml/forkexecd/test/fe_test.sh b/ocaml/forkexecd/test/fe_test.sh index fa5ffc514cd..aa0b9899ee7 100755 --- a/ocaml/forkexecd/test/fe_test.sh +++ b/ocaml/forkexecd/test/fe_test.sh @@ -6,13 +6,14 @@ export XDG_RUNTIME_DIR=${XDG_RUNTIME_DIR:-$TMPDIR} export FE_TEST=1 SOCKET=${XDG_RUNTIME_DIR}/xapi/forker/main +rm -f "$SOCKET" ../src/fe_main.exe & MAIN=$! cleanup () { kill $MAIN } -trap cleanup EXIT +trap cleanup EXIT INT for _ in $(seq 1 10); do test -S ${SOCKET} || sleep 1 done From bf93ab37f6b671630511fbe60744d2f50eef5254 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Fri, 5 Apr 2024 09:37:49 +0000 Subject: [PATCH 064/149] Add test for output strings Make sure we get the correct output from program launched. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/test/fe_test.ml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ocaml/forkexecd/test/fe_test.ml b/ocaml/forkexecd/test/fe_test.ml index e6c4ae21017..4c911498db5 100644 --- a/ocaml/forkexecd/test/fe_test.ml +++ b/ocaml/forkexecd/test/fe_test.ml @@ -140,6 +140,10 @@ let fail x = Printf.fprintf stderr "%s\n" x ; assert false +let expect expected s = + if s <> expected ^ "\n" then + fail (Printf.sprintf "output %s expected %s" s expected) + let test_exitcode () = let run_expect cmd expected = try Forkhelpers.execute_command_get_output cmd [] |> ignore @@ -154,12 +158,24 @@ let test_exitcode () = run_expect "/etc/hosts" 126 ; Printf.printf "\nCompleted exitcode tests\n" +let test_output () = + let exe = Printf.sprintf "/proc/%d/exe" (Unix.getpid ()) in + let expected_out = "output string" in + let expected_err = "error string" in + let args = ["echo"; expected_out; expected_err] in + let out, err = Forkhelpers.execute_command_get_output exe args in + expect expected_out out ; + expect expected_err err ; + print_endline "Completed output tests" + let master fds = Printf.printf "\nPerforming timeout tests\n%!" ; test_delay () ; test_notimeout () ; Printf.printf "\nCompleted timeout test\n%!" ; test_exitcode () ; + Printf.printf "\nPerforming input/output tests\n%!" ; + test_output () ; let combinations = shuffle (all_combinations fds) in Printf.printf "Starting %d tests\n%!" (List.length combinations) ; let i = ref 0 in @@ -236,6 +252,10 @@ let slave = function let sleep () = Unix.sleep 5 ; Printf.printf "Ok\n" +let echo out err = + if out <> "" then print_endline out ; + if err <> "" then prerr_endline err + let usage () = Printf.printf "Usage:\n" ; Printf.printf @@ -254,6 +274,8 @@ let _ = sleep () | _ :: "slave" :: rest -> slave rest + | _ :: "echo" :: out :: err :: _ -> + echo out err | [_] -> master max_fds | [_; fds] -> ( From 1ce582aba7e129bffdf3681816eab10c37db3a3e Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Fri, 5 Apr 2024 09:50:22 +0000 Subject: [PATCH 065/149] Add test for input strings Make sure program receives the string we are sending. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/test/fe_test.ml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ocaml/forkexecd/test/fe_test.ml b/ocaml/forkexecd/test/fe_test.ml index 4c911498db5..1df62eadc28 100644 --- a/ocaml/forkexecd/test/fe_test.ml +++ b/ocaml/forkexecd/test/fe_test.ml @@ -168,6 +168,16 @@ let test_output () = expect expected_err err ; print_endline "Completed output tests" +let test_input () = + let exe = Printf.sprintf "/proc/%d/exe" (Unix.getpid ()) in + let input = "input string" in + let args = ["replay"] in + let out, _ = + Forkhelpers.execute_command_get_output_send_stdin exe args input + in + expect input out ; + print_endline "Completed input tests" + let master fds = Printf.printf "\nPerforming timeout tests\n%!" ; test_delay () ; @@ -176,6 +186,7 @@ let master fds = test_exitcode () ; Printf.printf "\nPerforming input/output tests\n%!" ; test_output () ; + test_input () ; let combinations = shuffle (all_combinations fds) in Printf.printf "Starting %d tests\n%!" (List.length combinations) ; let i = ref 0 in @@ -256,6 +267,10 @@ let echo out err = if out <> "" then print_endline out ; if err <> "" then prerr_endline err +let replay () = + let line = read_line () in + print_endline line + let usage () = Printf.printf "Usage:\n" ; Printf.printf @@ -276,6 +291,8 @@ let _ = slave rest | _ :: "echo" :: out :: err :: _ -> echo out err + | _ :: "replay" :: _ -> + replay () | [_] -> master max_fds | [_; fds] -> ( From 53d5ae81ece77f6348c682f92795db4fdefc1a45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Wed, 7 Feb 2024 16:28:59 +0000 Subject: [PATCH 066/149] build: add sdk-build-c Makefile rule to test building C SDK locally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Requires libxml2-devel installed. Signed-off-by: Edwin Török (cherry picked from commit 000839541a4d1c266aad5222f5c1520ddb28ac2e) --- Makefile | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Makefile b/Makefile index 991ce87c812..d7f3db993be 100644 --- a/Makefile +++ b/Makefile @@ -117,6 +117,16 @@ sdk: sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/csharp sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/powershell +.PHONY: sdk-build-c sdk sdksanity + +sdk-build-c: sdk + cd _build/install/default/xapi/sdk/c && make -j $(JOBS) + +# workaround for no .resx generation, just for compilation testing +sdksanity: sdk + sed -i 's/FriendlyErrorNames.ResourceManager/null/g' ./_build/install/default/xapi/sdk/csharp/src/Failure.cs + cd _build/install/default/xapi/sdk/csharp/src && dotnet add package Newtonsoft.Json && dotnet build -f netstandard2.0 + .PHONY: sdk-build-java sdk-build-java: sdk From 484b098bb080679a6cf673b97e4da35ce5f0af91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Wed, 7 Feb 2024 15:15:44 +0000 Subject: [PATCH 067/149] CA_388624: fix(C SDK): fix build failure with recent GCC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The C SDK build was failing with a recent GCC on Fedora39 like this: ``` src/xen_common.c: In function ‘xen_session_logout’: src/xen_common.c:298:5: error: ‘xen_call_’ accessing 16 bytes in a region of size 0 [-Werror=stringop-overflow=] 298 | xen_call_(session, "session.logout", params, 0, NULL, NULL); | ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``` Use `NULL` instead of a VLA of size 0. Signed-off-by: Edwin Török (cherry picked from commit bfe78bdf03ef508bc3f56428dcb252793bd700df) --- ocaml/sdk-gen/c/autogen/src/xen_common.c | 21 ++++----------------- ocaml/sdk-gen/c/gen_c_binding.ml | 4 +++- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/ocaml/sdk-gen/c/autogen/src/xen_common.c b/ocaml/sdk-gen/c/autogen/src/xen_common.c index 9081f9bd725..9178d3fd43f 100644 --- a/ocaml/sdk-gen/c/autogen/src/xen_common.c +++ b/ocaml/sdk-gen/c/autogen/src/xen_common.c @@ -292,10 +292,7 @@ set_api_version(xen_session *session) void xen_session_logout(xen_session *session) { - abstract_value params[] = - { - }; - xen_call_(session, "session.logout", params, 0, NULL, NULL); + xen_call_(session, "session.logout", NULL, 0, NULL, NULL); if (session->error_description != NULL) { @@ -314,10 +311,7 @@ xen_session_logout(xen_session *session) void xen_session_local_logout(xen_session *session) { - abstract_value params[] = - { - }; - xen_call_(session, "session.local_logout", params, 0, NULL, NULL); + xen_call_(session, "session.local_logout", NULL, 0, NULL, NULL); if (session->error_description != NULL) { @@ -336,14 +330,11 @@ xen_session_local_logout(xen_session *session) bool xen_session_get_all_subject_identifiers(xen_session *session, struct xen_string_set **result) { - abstract_value params[] = - { - }; abstract_type result_type = abstract_type_string_set; *result = NULL; - xen_call_(session, "session.get_all_subject_identifiers", params, 0, &result_type, result); + xen_call_(session, "session.get_all_subject_identifiers", NULL, 0, &result_type, result); return session->ok; } @@ -351,14 +342,10 @@ bool bool xen_session_get_all_subject_identifiers_async(xen_session *session, xen_task *result) { - abstract_value params[] = - { - }; - abstract_type result_type = abstract_type_string; *result = NULL; - xen_call_(session, "Async.session.get_all_subject_identifiers", params, 0, &result_type, result); + xen_call_(session, "Async.session.get_all_subject_identifiers", NULL, 0, &result_type, result); return session->ok; } diff --git a/ocaml/sdk-gen/c/gen_c_binding.ml b/ocaml/sdk-gen/c/gen_c_binding.ml index e9035a88c7f..2302fc6cf99 100644 --- a/ocaml/sdk-gen/c/gen_c_binding.ml +++ b/ocaml/sdk-gen/c/gen_c_binding.ml @@ -374,7 +374,9 @@ and impl_message needed classname message = sprintf " xen_call_(session, \"%s.%s\", %s, %d, NULL, NULL);\n\ \ return session->ok;\n" - classname message.msg_name param_call param_count + classname message.msg_name + (if param_count = 0 then "NULL" else param_call) + param_count in let messageAsyncImpl = impl_message_async needed classname message in From 14052b202b3a2864fd1c6b5e484c6f6349d4c3f9 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 8 Apr 2024 13:20:57 +0100 Subject: [PATCH 068/149] CA-390988: Prevent varstored-guard from shutting down while domains run Domains using TPMs need to have SWTPM always available. Encode this information in the service file so the services are shut down at the correct time while powering off the hosts. Signed-off-by: Pau Ruiz Safont --- scripts/varstored-guard.service | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/varstored-guard.service b/scripts/varstored-guard.service index 59e45110f97..c9d1b9bd939 100644 --- a/scripts/varstored-guard.service +++ b/scripts/varstored-guard.service @@ -2,7 +2,7 @@ Description=Varstored XAPI socket deprivileging daemon Documentation=man:varstored-guard(1) After=message-switch.service syslog.target -Before=xenopsd.service +Before=xapi-domains.service xenopsd.service Wants=message-switch.service syslog.target [Service] From f8f3d6198878e8d2dde265695c022ae646d4540a Mon Sep 17 00:00:00 2001 From: Rob Hoes Date: Mon, 4 Dec 2023 15:17:24 +0000 Subject: [PATCH 069/149] CP-46851: Allow metadata-exports of snapshots I am not sure where this restriction came from. It was introduced in the same commit that introduced the metadata-export feature itself, without explanation. Full exports of snapshots are already allowed. Signed-off-by: Rob Hoes --- ocaml/xapi/export.ml | 10 ---------- quality-gate.sh | 2 +- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/ocaml/xapi/export.ml b/ocaml/xapi/export.ml index 49ccc7b0c57..93572247076 100644 --- a/ocaml/xapi/export.ml +++ b/ocaml/xapi/export.ml @@ -771,16 +771,6 @@ let metadata_handler (req : Request.t) s _ = else [vm_from_request ~__context req] in - if - (not export_all) - && Db.VM.get_is_a_snapshot ~__context ~self:(List.hd vm_refs) - then - raise - (Api_errors.Server_error - ( Api_errors.operation_not_allowed - , ["Exporting metadata of a snapshot is not allowed"] - ) - ) ; let task_id = Ref.string_of (Context.get_task_id __context) in let read_fd, write_fd = Unix.pipe () in let export_error = ref None in diff --git a/quality-gate.sh b/quality-gate.sh index 77238f4ab93..ffbe1745d23 100755 --- a/quality-gate.sh +++ b/quality-gate.sh @@ -3,7 +3,7 @@ set -e list-hd () { - N=318 + N=317 LIST_HD=$(git grep -r --count 'List.hd' -- **/*.ml | cut -d ':' -f 2 | paste -sd+ - | bc) if [ "$LIST_HD" -eq "$N" ]; then echo "OK counted $LIST_HD List.hd usages" From 1141b8a9a448cdc4b1bed518665ff72f545e6e26 Mon Sep 17 00:00:00 2001 From: Rob Hoes Date: Tue, 9 Jan 2024 17:01:43 +0000 Subject: [PATCH 070/149] CP-46851: return snapshot refs when importing exclusively snapshots This allows clients to know which snapshots they knowingly just created and act on them instead of looking again into the db and try to divine which snapshots the call created. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi/import.ml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ocaml/xapi/import.ml b/ocaml/xapi/import.ml index fd6d898b1e0..01e5ca25640 100644 --- a/ocaml/xapi/import.ml +++ b/ocaml/xapi/import.ml @@ -2158,11 +2158,18 @@ let complete_import ~__context vmrefs = Xapi_vm_lifecycle.update_allowed_operations ~__context ~self:vm ) vmrefs ; - (* We only keep VMs which are not snapshot *) + (* When only snapshots have been imported, return all of them. + Otherwise, only keep VMs which are not snapshots *) let vmrefs = - List.filter - (fun vmref -> not (Db.VM.get_is_a_snapshot ~__context ~self:vmref)) + let non_snapshots = + List.filter + (fun x -> not (Db.VM.get_is_a_snapshot ~__context ~self:x)) + vmrefs + in + if non_snapshots = [] then vmrefs + else + non_snapshots in (* We only set the result on the task since it is officially completed later. *) TaskHelper.set_result ~__context (Some (API.rpc_of_ref_VM_set vmrefs)) From e9431c6a003d93ee511e582ee73a288663f7aec0 Mon Sep 17 00:00:00 2001 From: Rob Hoes Date: Tue, 16 Jan 2024 10:36:34 +0000 Subject: [PATCH 071/149] CP-46851: Add metadata_export to VM.allowed_operations This allows clients to detect that the newer device type exclusions for export are available. Signed-off-by: Rob Hoes --- ocaml/xapi/xapi_vm_lifecycle.ml | 1 + 1 file changed, 1 insertion(+) diff --git a/ocaml/xapi/xapi_vm_lifecycle.ml b/ocaml/xapi/xapi_vm_lifecycle.ml index d90da39619e..7d35a12f1d0 100644 --- a/ocaml/xapi/xapi_vm_lifecycle.ml +++ b/ocaml/xapi/xapi_vm_lifecycle.ml @@ -790,6 +790,7 @@ let update_allowed_operations ~__context ~self = ; `changing_dynamic_range ; `changing_NVRAM ; `create_vtpm + ; `metadata_export ] in (* FIXME: need to be able to deal with rolling-upgrade for orlando as well *) From f48b8787675da239eb6d839ae86f26c2464eedc2 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 19 Mar 2024 17:34:46 +0000 Subject: [PATCH 072/149] CP-46851: add parameter to skip device types on get_export_metadata This allows to orchestrators to copy devices like VBDs from the VM, export the metadata without the disk, import the VM to any pool, and finally add the saved VBDs to the new VM before booting it. Signed-off-by: Pau Ruiz Safont --- ocaml/idl/datamodel.ml | 1 + ocaml/xapi/export.ml | 127 ++++++++++++++++++++----------------- ocaml/xapi/importexport.ml | 31 +++++++++ ocaml/xapi/xapi_dr.ml | 1 + 4 files changed, 102 insertions(+), 58 deletions(-) diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index c8fa2614150..a9f219e91af 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -8177,6 +8177,7 @@ let http_actions = ; Bool_query_arg "include_dom0" ; Bool_query_arg "include_vhd_parents" ; Bool_query_arg "export_snapshots" + ; String_query_arg "excluded_device_types" ] , _R_VM_ADMIN , [] diff --git a/ocaml/xapi/export.ml b/ocaml/xapi/export.ml index 93572247076..54a494ac240 100644 --- a/ocaml/xapi/export.ml +++ b/ocaml/xapi/export.ml @@ -52,7 +52,7 @@ let make_id = "Ref:" ^ string_of_int this let rec update_table ~__context ~include_snapshots ~preserve_power_state - ~include_vhd_parents ~table vm = + ~include_vhd_parents ~table ~excluded_devices vm = let add r = if not (Hashtbl.mem table (Ref.string_of r)) then Hashtbl.add table (Ref.string_of r) (make_id ()) @@ -77,38 +77,40 @@ let rec update_table ~__context ~include_snapshots ~preserve_power_state then ( add vm ; let vm = Db.VM.get_record ~__context ~self:vm in - List.iter - (fun vif -> - if Db.is_valid_ref __context vif then ( - add vif ; - let vif = Db.VIF.get_record ~__context ~self:vif in - add vif.API.vIF_network + if not (List.mem Devicetype.VIF excluded_devices) then + List.iter + (fun vif -> + if Db.is_valid_ref __context vif then ( + add vif ; + let vif = Db.VIF.get_record ~__context ~self:vif in + add vif.API.vIF_network + ) ) - ) - vm.API.vM_VIFs ; - List.iter - (fun vbd -> - if Db.is_valid_ref __context vbd then ( - add vbd ; - let vbd = Db.VBD.get_record ~__context ~self:vbd in - if not vbd.API.vBD_empty then - add_vdi vbd.API.vBD_VDI + vm.API.vM_VIFs ; + if not (List.mem Devicetype.VBD excluded_devices) then + List.iter + (fun vbd -> + if Db.is_valid_ref __context vbd then ( + add vbd ; + let vbd = Db.VBD.get_record ~__context ~self:vbd in + if not vbd.API.vBD_empty then + add_vdi vbd.API.vBD_VDI + ) ) - ) - vm.API.vM_VBDs ; - List.iter - (fun vgpu -> - if Db.is_valid_ref __context vgpu then ( - add vgpu ; - let vgpu = Db.VGPU.get_record ~__context ~self:vgpu in - add vgpu.API.vGPU_type ; - add vgpu.API.vGPU_GPU_group + vm.API.vM_VBDs ; + if not (List.mem Devicetype.VGPU excluded_devices) then + List.iter + (fun vgpu -> + if Db.is_valid_ref __context vgpu then ( + add vgpu ; + let vgpu = Db.VGPU.get_record ~__context ~self:vgpu in + add vgpu.API.vGPU_type ; + add vgpu.API.vGPU_GPU_group + ) ) - ) - vm.API.vM_VGPUs ; + vm.API.vM_VGPUs ; (* add all PVS proxies that have a VIF belonging to this VM, add their - * PVS sites as well - *) + PVS sites as well *) Db.PVS_proxy.get_all_records ~__context |> List.filter (fun (_, p) -> List.mem p.API.pVS_proxy_VIF vm.API.vM_VIFs) |> List.iter (fun (ref, proxy) -> @@ -118,15 +120,16 @@ let rec update_table ~__context ~include_snapshots ~preserve_power_state ) ) ; (* add VTPMs that belong to this VM *) - vm.API.vM_VTPMs - |> List.iter (fun ref -> if Db.is_valid_ref __context ref then add ref) ; + if not (List.mem Devicetype.VTPM excluded_devices) then + vm.API.vM_VTPMs + |> List.iter (fun ref -> if Db.is_valid_ref __context ref then add ref) ; (* If we need to include snapshots, update the table for VMs in the 'snapshots' field *) if include_snapshots then List.iter (fun snap -> update_table ~__context ~include_snapshots:false ~preserve_power_state - ~include_vhd_parents ~table snap + ~include_vhd_parents ~table ~excluded_devices snap ) vm.API.vM_snapshots ; (* If VM is suspended then add the suspend_VDI *) @@ -145,7 +148,7 @@ let rec update_table ~__context ~include_snapshots ~preserve_power_state (* Add the parent VM *) if include_snapshots && Db.is_valid_ref __context vm.API.vM_parent then update_table ~__context ~include_snapshots:false ~preserve_power_state - ~include_vhd_parents ~table vm.API.vM_parent + ~include_vhd_parents ~table ~excluded_devices vm.API.vM_parent ) (** Walk the graph of objects and update the table of Ref -> ids for each object we wish @@ -580,11 +583,11 @@ let make_all ~with_snapshot_metadata ~preserve_power_state table __context = on metadata-export, include snapshots fields of the exported VM as well as the VM records of VMs which are snapshots of the exported VM. *) let vm_metadata ~with_snapshot_metadata ~preserve_power_state - ~include_vhd_parents ~__context ~vms = + ~include_vhd_parents ~__context ~vms ~excluded_devices = let table = create_table () in List.iter (update_table ~__context ~include_snapshots:with_snapshot_metadata - ~preserve_power_state ~include_vhd_parents ~table + ~preserve_power_state ~include_vhd_parents ~table ~excluded_devices ) vms ; let objects = @@ -603,7 +606,7 @@ let string_of_vm ~__context vm = (** Export a VM's metadata only *) let export_metadata ~__context ~with_snapshot_metadata ~preserve_power_state - ~include_vhd_parents ~vms s = + ~include_vhd_parents ~vms ~excluded_devices s = ( match vms with | [] -> failwith "need to specify at least one VM" @@ -624,7 +627,7 @@ let export_metadata ~__context ~with_snapshot_metadata ~preserve_power_state ) ; let _, ova_xml = vm_metadata ~with_snapshot_metadata ~preserve_power_state - ~include_vhd_parents ~__context ~vms + ~include_vhd_parents ~__context ~vms ~excluded_devices in let hdr = Tar.Header.make Xapi_globs.ova_xml_filename @@ -640,7 +643,7 @@ let export refresh_session __context rpc session_id s vm_ref (string_of_bool preserve_power_state) ; let table, ova_xml = vm_metadata ~with_snapshot_metadata:false ~preserve_power_state - ~include_vhd_parents:false ~__context ~vms:[vm_ref] + ~include_vhd_parents:false ~__context ~vms:[vm_ref] ~excluded_devices:[] in debug "Outputting ova.xml" ; let hdr = @@ -716,35 +719,43 @@ let vm_from_request ~__context (req : Request.t) = Client.VM.get_by_uuid ~rpc ~session_id ~uuid ) -let bool_from_request ~__context (req : Request.t) default k = - if List.mem_assoc k req.Request.query then - bool_of_string (List.assoc k req.Request.query) - else - default +let arg_from_request (req : Request.t) k = List.assoc_opt k req.Request.query -let export_all_vms_from_request ~__context (req : Request.t) = - bool_from_request ~__context req false "all" +let bool_from_request req default k = + arg_from_request req k |> Option.fold ~none:default ~some:bool_of_string + +let devicetypelist_from_request req default k = + let to_list = function + | "" -> + [] + | x -> + String.split_on_char ',' x |> List.map Devicetype.of_string + in + arg_from_request req k |> Option.fold ~none:default ~some:to_list -let include_vhd_parents_from_request ~__context (req : Request.t) = - bool_from_request ~__context req false "include_vhd_parents" +let export_all_vms_from_request req = bool_from_request req false "all" -let export_snapshots_from_request ~__context (req : Request.t) = - bool_from_request ~__context req true "export_snapshots" +let include_vhd_parents_from_request req = + bool_from_request req false "include_vhd_parents" -let include_dom0_from_request ~__context (req : Request.t) = - bool_from_request ~__context req true "include_dom0" +let export_snapshots_from_request req = + bool_from_request req true "export_snapshots" + +let include_dom0_from_request req = bool_from_request req true "include_dom0" + +let excluded_devices_from_request req = + devicetypelist_from_request req [] "excluded_device_types" let metadata_handler (req : Request.t) s _ = debug "metadata_handler called" ; req.Request.close <- true ; (* Xapi_http.with_context always completes the task at the end *) Xapi_http.with_context "VM.export_metadata" req s (fun __context -> - let include_vhd_parents = - include_vhd_parents_from_request ~__context req - in - let export_all = export_all_vms_from_request ~__context req in - let export_snapshots = export_snapshots_from_request ~__context req in - let include_dom0 = include_dom0_from_request ~__context req in + let include_vhd_parents = include_vhd_parents_from_request req in + let export_all = export_all_vms_from_request req in + let export_snapshots = export_snapshots_from_request req in + let include_dom0 = include_dom0_from_request req in + let excluded_devices = excluded_devices_from_request req in (* Get the VM refs. In case of exporting the metadata of a particular VM, return a singleton list containing the vm ref. *) (* In case of exporting all the VMs metadata, get all the VM records which are not default templates. *) let vm_refs = @@ -790,7 +801,7 @@ let metadata_handler (req : Request.t) s _ = vm_refs ; export_metadata ~with_snapshot_metadata:export_snapshots ~preserve_power_state:true ~include_vhd_parents - ~__context ~vms:vm_refs write_fd + ~excluded_devices ~__context ~vms:vm_refs write_fd ) (fun () -> Unix.close write_fd ; diff --git a/ocaml/xapi/importexport.ml b/ocaml/xapi/importexport.ml index a7354fce45e..f90a8da80ea 100644 --- a/ocaml/xapi/importexport.ml +++ b/ocaml/xapi/importexport.ml @@ -469,6 +469,37 @@ module Format = struct (* default *) end +module Devicetype = struct + type t = VIF | VBD | VGPU | VTPM + + let all = [VIF; VBD; VGPU; VTPM] + + let to_string = function + | VIF -> + "vif" + | VBD -> + "vbd" + | VGPU -> + "vgpu" + | VTPM -> + "vtpm" + + let of_string x = + match String.lowercase_ascii x with + | "vif" -> + VIF + | "vbd" -> + VBD + | "vgpu" -> + VGPU + | "vtpm" -> + VTPM + | other -> + let fail fmt = Printf.kprintf failwith fmt in + fail "%s: Type '%s' not one of [%s]" __FUNCTION__ other + (String.concat "; " (List.map to_string all)) +end + let return_302_redirect (req : Http.Request.t) s address = let address = Http.Url.maybe_wrap_IPv6_literal address in let url = diff --git a/ocaml/xapi/xapi_dr.ml b/ocaml/xapi/xapi_dr.ml index e9c1c53ad0c..bdbb4dee6c2 100644 --- a/ocaml/xapi/xapi_dr.ml +++ b/ocaml/xapi/xapi_dr.ml @@ -245,6 +245,7 @@ let create_import_objects ~__context ~vms = List.iter (Export.update_table ~__context ~include_snapshots:true ~preserve_power_state:true ~include_vhd_parents:false ~table + ~excluded_devices:[] ) vms ; Export.make_all ~with_snapshot_metadata:true ~preserve_power_state:true table From e67585c58d40cc52fc4a93aa22a9d75879e45e69 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Thu, 21 Mar 2024 13:53:07 +0000 Subject: [PATCH 073/149] CP-46851: Add device types excluded to exports in xe-cli This allows to easily test the http endpoint. Also advertises the metadata parameter on these calls Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-cli-server/cli_frontend.ml | 18 +++++++++++++++--- ocaml/xapi-cli-server/cli_operations.ml | 13 ++++++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/ocaml/xapi-cli-server/cli_frontend.ml b/ocaml/xapi-cli-server/cli_frontend.ml index f8aa043eb5a..58ecb6cc88c 100644 --- a/ocaml/xapi-cli-server/cli_frontend.ml +++ b/ocaml/xapi-cli-server/cli_frontend.ml @@ -1756,7 +1756,13 @@ let rec cmdtable_data : (string * cmd_spec) list = ; ( "vm-export" , { reqd= ["filename"] - ; optn= ["preserve-power-state"; "compress"] + ; optn= + [ + "preserve-power-state" + ; "compress" + ; "metadata" + ; "excluded-device-types" + ] ; help= "Export a VM to ." ; implementation= With_fd Cli_operations.vm_export ; flags= [Standard; Vm_selectors] @@ -1798,7 +1804,13 @@ let rec cmdtable_data : (string * cmd_spec) list = ; ( "snapshot-export-to-template" , { reqd= ["filename"; "snapshot-uuid"] - ; optn= ["preserve-power-state"] + ; optn= + [ + "preserve-power-state" + ; "compress" + ; "metadata" + ; "excluded-device-types" + ] ; help= "Export a snapshot to ." ; implementation= With_fd Cli_operations.snapshot_export ; flags= [Standard] @@ -1863,7 +1875,7 @@ let rec cmdtable_data : (string * cmd_spec) list = ; ( "template-export" , { reqd= ["filename"; "template-uuid"] - ; optn= [] + ; optn= ["compress"; "metadata"; "excluded-device-types"] ; help= "Export a template to ." ; implementation= With_fd Cli_operations.template_export ; flags= [Standard] diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index bc0d9ea30bc..27a62085453 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -5867,7 +5867,13 @@ let export_common fd _printer rpc session_id params filename num ?task_uuid else vm_metadata_only in - let vm_metadata_only = get_bool_param params "metadata" in + let extra_args = + if vm_metadata_only then + Printf.sprintf "&excluded_device_types=%s" + (get_param params ~default:"" "excluded-device-types") + else + "" + in let vm_record = vm.record () in let exporttask, task_destroy_fn = match task_uuid with @@ -5893,7 +5899,7 @@ let export_common fd _printer rpc session_id params filename num ?task_uuid let f = if !num > 1 then filename ^ string_of_int !num else filename in download_file rpc session_id exporttask fd f (Printf.sprintf - "%s?session_id=%s&task_id=%s&ref=%s&%s=%s&preserve_power_state=%b&export_snapshots=%b" + "%s?session_id=%s&task_id=%s&ref=%s&%s=%s&preserve_power_state=%b&export_snapshots=%b%s" ( if vm_metadata_only then Constants.export_metadata_uri else @@ -5903,7 +5909,7 @@ let export_common fd _printer rpc session_id params filename num ?task_uuid (Ref.string_of (vm.getref ())) Constants.use_compression (Compression_algorithms.to_string compression) - preserve_power_state export_snapshots + preserve_power_state export_snapshots extra_args ) "Export" ; num := !num + 1 @@ -5939,6 +5945,7 @@ let vm_export fd printer rpc session_id params = ; "compress" ; "preserve-power-state" ; "include-snapshots" + ; "excluded-device-types" ] ) From d772c3904dc516edb75e539d528357139a6a02ec Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 20 Mar 2024 15:41:39 +0000 Subject: [PATCH 074/149] xapi-cli-server: simplify vm-export and helpers Use the helper functions to process parameters, use atomics instead of ad-hoc references for counting in possibly-parallel case Signed-off-by: Pau Ruiz Safont --- ocaml/xapi-cli-server/cli_operations.ml | 218 +++++++++--------------- ocaml/xapi-cli-server/record_util.ml | 11 +- quality-gate.sh | 2 +- 3 files changed, 91 insertions(+), 140 deletions(-) diff --git a/ocaml/xapi-cli-server/cli_operations.ml b/ocaml/xapi-cli-server/cli_operations.ml index 27a62085453..4f28a48848d 100644 --- a/ocaml/xapi-cli-server/cli_operations.ml +++ b/ocaml/xapi-cli-server/cli_operations.ml @@ -32,26 +32,10 @@ let failwith str = raise (Cli_util.Cli_failure str) exception ExitWithError of int let bool_of_string param string = - let s = String.lowercase_ascii string in - match s with - | "true" -> - true - | "t" -> - true - | "1" -> - true - | "false" -> - false - | "f" -> - false - | "0" -> - false - | _ -> - failwith - ("Failed to parse parameter '" - ^ param - ^ "': expecting 'true' or 'false'" - ) + try Record_util.bool_of_string string + with Record_util.Record_failure msg -> + let msg = Printf.sprintf "Failed to parse parameter '%s': %s" param msg in + raise (Record_util.Record_failure msg) let get_bool_param params ?(default = false) param = List.assoc_opt param params @@ -66,6 +50,24 @@ let get_float_param params param ~default = let get_param params param ~default = Option.value ~default (List.assoc_opt param params) +let get_set_param params ?(default = []) param = + List.assoc_opt param params + |> Option.map (String.split_on_char ',') + |> Option.value ~default + +let get_map_param params ?(default = []) param = + let get_map x = + String.split_on_char ',' x + |> List.filter_map (fun x -> + match String.split_on_char ':' x with + | [k; v] -> + Some (k, v) + | _ -> + None + ) + in + List.assoc_opt param params |> Option.map get_map |> Option.value ~default + (** [get_unique_param param params] is intended to replace [List.assoc_opt] in the cases where a parameter can only exist once, as repeating it might force the CLI to make choices the user didn't foresee. In those cases @@ -1520,16 +1522,15 @@ let pool_management_reconfigure (_ : printer) rpc session_id params = let pool_join printer rpc session_id params = try let force = get_bool_param params "force" in + let master_address = List.assoc "master-address" params in + let master_username = List.assoc "master-username" params in + let master_password = List.assoc "master-password" params in if force then - Client.Pool.join_force ~rpc ~session_id - ~master_address:(List.assoc "master-address" params) - ~master_username:(List.assoc "master-username" params) - ~master_password:(List.assoc "master-password" params) + Client.Pool.join_force ~rpc ~session_id ~master_address ~master_username + ~master_password else - Client.Pool.join ~rpc ~session_id - ~master_address:(List.assoc "master-address" params) - ~master_username:(List.assoc "master-username" params) - ~master_password:(List.assoc "master-password" params) ; + Client.Pool.join ~rpc ~session_id ~master_address ~master_username + ~master_password ; printer (Cli_printer.PList [ @@ -3264,11 +3265,11 @@ let do_vm_op ?(include_control_vms = false) ?(include_template_vms = false) select_vms ~include_control_vms ~include_template_vms rpc session_id params ignore_params in - match List.length vms with - | 0 -> + match vms with + | [] -> failwith "No matching VMs found" - | 1 -> - [op (List.hd vms)] + | [vm] -> + [op vm] | _ -> if multiple && get_bool_param params "multiple" then do_multiple op vms @@ -3310,11 +3311,11 @@ let do_host_op rpc session_id op params ?(multiple = true) ignore_params = let do_sr_op rpc session_id op params ?(multiple = true) ignore_params = let srs = select_srs rpc session_id params ignore_params in - match List.length srs with - | 0 -> + match srs with + | [] -> failwith "No matching hosts found" - | 1 -> - [op (List.hd srs)] + | [sr] -> + [op sr] | _ -> if multiple && get_bool_param params "multiple" then do_multiple op srs @@ -5575,12 +5576,7 @@ let vm_import fd _printer rpc session_id params = raise (Cli_util.Cli_failure "No SR specified and Pool default SR is null") in - let _type = - if List.mem_assoc "type" params then - List.assoc "type" params - else - "default" - in + let _type = get_param ~default:"default" params "type" in let full_restore = get_bool_param params "preserve" in let vm_metadata_only = get_bool_param params "metadata" in let force = get_bool_param params "force" in @@ -5806,9 +5802,7 @@ let blob_put fd _printer rpc session_id params = let blob_create printer rpc session_id params = let name = List.assoc "name" params in let mime_type = Listext.assoc_default "mime-type" params "" in - let public = - try bool_of_string "public" (List.assoc "public" params) with _ -> false - in + let public = get_bool_param params "public" in if List.mem_assoc "vm-uuid" params then let uuid = List.assoc "vm-uuid" params in let vm = Client.VM.get_by_uuid ~rpc ~session_id ~uuid in @@ -5860,19 +5854,16 @@ let blob_create printer rpc session_id params = let export_common fd _printer rpc session_id params filename num ?task_uuid compression preserve_power_state vm = - let vm_metadata_only : bool = get_bool_param params "metadata" in - let export_snapshots : bool = - if List.mem_assoc "include-snapshots" params then - bool_of_string "include-snapshots" (List.assoc "include-snapshots" params) - else - vm_metadata_only - in - let extra_args = + let vm_metadata_only = get_bool_param params "metadata" in + let export_snapshots = get_bool_param params "include-snapshots" in + let uri, extra_args = if vm_metadata_only then - Printf.sprintf "&excluded_device_types=%s" - (get_param params ~default:"" "excluded-device-types") + ( Constants.export_metadata_uri + , Printf.sprintf "&excluded_device_types=%s" + (get_param params ~default:"" "excluded-device-types") + ) else - "" + (Constants.export_uri, "") in let vm_record = vm.record () in let exporttask, task_destroy_fn = @@ -5890,49 +5881,40 @@ let export_common fd _printer rpc session_id params filename num ?task_uuid (* do not destroy the task that has been received *) (Client.Task.get_by_uuid ~rpc ~session_id ~uuid:task_uuid, fun () -> ()) in - (* Initially mark the task progress as -1.0. The first thing the export handler does it to mark it as zero *) - (* This is used as a flag to show that the 'ownership' of the task has been passed to the handler, and it's *) - (* not our responsibility any more to mark the task as completed/failed/etc. *) + (* Initially mark the task progress as -1.0. The first thing the export + handler does it to mark it as zero. This is used as a flag to show that + the 'ownership' of the task has been passed to the handler, and it's + not our responsibility any more to mark the task as completed/failed/etc. + *) Client.Task.set_progress ~rpc ~session_id ~self:exporttask ~value:(-1.0) ; finally (fun () -> - let f = if !num > 1 then filename ^ string_of_int !num else filename in + let num = Atomic.fetch_and_add num 1 in + let f = if num > 1 then filename ^ string_of_int num else filename in download_file rpc session_id exporttask fd f (Printf.sprintf "%s?session_id=%s&task_id=%s&ref=%s&%s=%s&preserve_power_state=%b&export_snapshots=%b%s" - ( if vm_metadata_only then - Constants.export_metadata_uri - else - Constants.export_uri - ) - (Ref.string_of session_id) (Ref.string_of exporttask) + uri (Ref.string_of session_id) (Ref.string_of exporttask) (Ref.string_of (vm.getref ())) Constants.use_compression (Compression_algorithms.to_string compression) preserve_power_state export_snapshots extra_args ) - "Export" ; - num := !num + 1 + "Export" ) (fun () -> task_destroy_fn ()) let get_compression_algorithm params = - if List.mem_assoc "compress" params then - Compression_algorithms.of_string (List.assoc "compress" params) - else - None + Option.bind + (List.assoc_opt "compress" params) + Compression_algorithms.of_string let vm_export fd printer rpc session_id params = let filename = List.assoc "filename" params in let compression = get_compression_algorithm params in let preserve_power_state = get_bool_param params "preserve-power-state" in - let task_uuid = - if List.mem_assoc "task-uuid" params then - Some (List.assoc "task-uuid" params) - else - None - in - let num = ref 1 in + let task_uuid = List.assoc_opt "task-uuid" params in + let num = Atomic.make 1 in let op vm = export_common fd printer rpc session_id params filename num ?task_uuid compression preserve_power_state vm @@ -5953,32 +5935,23 @@ let vm_export_aux obj_type fd printer rpc session_id params = let filename = List.assoc "filename" params in let compression = get_compression_algorithm params in let preserve_power_state = get_bool_param params "preserve-power-state" in - let num = ref 1 in let uuid = List.assoc (obj_type ^ "-uuid") params in - let ref = Client.VM.get_by_uuid ~rpc ~session_id ~uuid in - if - obj_type = "template" - && not (Client.VM.get_is_a_template ~rpc ~session_id ~self:ref) - then - failwith - (Printf.sprintf - "This operation can only be performed on a VM template. %s is not a \ - VM template." - uuid - ) ; - if - obj_type = "snapshot" - && not (Client.VM.get_is_a_snapshot ~rpc ~session_id ~self:ref) - then - failwith - (Printf.sprintf - "This operation can only be performed on a VM snapshot. %s is not a \ - VM snapshot." - uuid - ) ; + let vm = Client.VM.get_by_uuid ~rpc ~session_id ~uuid in + let is_template () = Client.VM.get_is_a_template ~rpc ~session_id ~self:vm in + let is_snapshot () = Client.VM.get_is_a_snapshot ~rpc ~session_id ~self:vm in + let msg () = + Printf.sprintf + "This operation can only be performed on a VM %s. %s is not a VM %s." + obj_type uuid obj_type + in + if obj_type = "template" && not (is_template ()) then + failwith (msg ()) ; + if obj_type = "snapshot" && not (is_snapshot ()) then + failwith (msg ()) ; + let num = Atomic.make 1 in export_common fd printer rpc session_id params filename num compression preserve_power_state - (vm_record rpc session_id ref) + (vm_record rpc session_id vm) let vm_copy_bios_strings printer rpc session_id params = let host = @@ -7356,7 +7329,7 @@ let vmss_create printer rpc session_id params = let schedule = read_map_params "schedule" params in (* optional parameters with default values *) let name_description = get "name-description" ~default:"" in - let enabled = Record_util.bool_of_string (get "enabled" ~default:"true") in + let enabled = get_bool_param ~default:true params "enabled" in let retained_snapshots = Int64.of_string (get "retained-snapshots" ~default:"7") in @@ -7925,13 +7898,7 @@ module VTPM = struct let create printer rpc session_id params = let vm_uuid = List.assoc "vm-uuid" params in let vM = Client.VM.get_by_uuid ~rpc ~session_id ~uuid:vm_uuid in - let is_unique = - match List.assoc_opt "is_unique" params with - | Some value -> - bool_of_string "is_unique" value - | None -> - false - in + let is_unique = get_bool_param params "is_unique" in let ref = Client.VTPM.create ~rpc ~session_id ~vM ~is_unique in let uuid = Client.VTPM.get_uuid ~rpc ~session_id ~self:ref in printer (Cli_printer.PList [uuid]) @@ -7947,33 +7914,12 @@ module Observer = struct let create printer rpc session_id params = let name_label = List.assoc "name-label" params in let hosts = - List.assoc_opt "host-uuids" params - |> Option.fold ~none:[] ~some:(fun host_uuids -> - List.map - (fun uuid -> Client.Host.get_by_uuid ~rpc ~session_id ~uuid) - (String.split_on_char ',' host_uuids) - ) - in - let name_description = - List.assoc_opt "name-description" params |> Option.value ~default:"" - in - let enabled = - List.assoc_opt "enabled" params - |> Option.fold ~none:false ~some:(fun s -> - try Stdlib.bool_of_string s with _ -> false - ) - in - let attributes = - List.assoc_opt "attributes" params - |> Option.fold ~none:[] ~some:(String.split_on_char ',') - |> List.filter_map (fun kv -> - match String.split_on_char ':' kv with - | [k; v] -> - Some (k, v) - | _ -> - None - ) + get_set_param params "host-uuids" + |> List.map (fun uuid -> Client.Host.get_by_uuid ~rpc ~session_id ~uuid) in + let name_description = get_param ~default:"" params "name-description" in + let enabled = get_bool_param params "enabled" in + let attributes = get_map_param params "attributes" in let endpoints = List.assoc_opt "endpoints" params |> Option.fold ~none:[Tracing.bugtool_name] diff --git a/ocaml/xapi-cli-server/record_util.ml b/ocaml/xapi-cli-server/record_util.ml index 5332c2aee16..8fbd141e908 100644 --- a/ocaml/xapi-cli-server/record_util.ml +++ b/ocaml/xapi-cli-server/record_util.ml @@ -953,12 +953,17 @@ let cluster_host_operation_to_string op = let bool_of_string s = match String.lowercase_ascii s with - | "true" | "yes" -> + | "true" | "t" | "yes" | "y" | "1" -> true - | "false" | "no" -> + | "false" | "f" | "no" | "n" | "0" -> false | _ -> - raise (Record_failure ("Expected 'true','yes','false','no', got " ^ s)) + raise + (Record_failure + ("Expected 'true','t','yes','y','1','false','f','no','n','0' got " + ^ s + ) + ) let sdn_protocol_of_string s = match String.lowercase_ascii s with diff --git a/quality-gate.sh b/quality-gate.sh index ffbe1745d23..56e53e75b56 100755 --- a/quality-gate.sh +++ b/quality-gate.sh @@ -3,7 +3,7 @@ set -e list-hd () { - N=317 + N=315 LIST_HD=$(git grep -r --count 'List.hd' -- **/*.ml | cut -d ':' -f 2 | paste -sd+ - | bc) if [ "$LIST_HD" -eq "$N" ]; then echo "OK counted $LIST_HD List.hd usages" From f237795fb1ec344c47587f8227f9d047adcf3ef1 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 19 Mar 2024 17:38:22 +0000 Subject: [PATCH 075/149] datamodel_vm: fix typo Signed-off-by: Pau Ruiz Safont --- ocaml/idl/datamodel_vm.ml | 2 +- ocaml/idl/schematest.ml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ocaml/idl/datamodel_vm.ml b/ocaml/idl/datamodel_vm.ml index c1a6b9a7d9c..aa45d93de5b 100644 --- a/ocaml/idl/datamodel_vm.ml +++ b/ocaml/idl/datamodel_vm.ml @@ -1899,7 +1899,7 @@ let t = ; field ~qualifier:DynamicRO ~ty:(Set (Ref _vbd)) "VBDs" "virtual block devices" ; field ~qualifier:DynamicRO ~ty:(Set (Ref _vusb)) "VUSBs" - "vitual usb devices" + "virtual usb devices" ; field ~writer_roles:_R_POOL_ADMIN ~qualifier:DynamicRO ~ty:(Set (Ref _crashdump)) "crash_dumps" "crash dumps associated with this VM" diff --git a/ocaml/idl/schematest.ml b/ocaml/idl/schematest.ml index c8e5972c9a6..82619e8393d 100644 --- a/ocaml/idl/schematest.ml +++ b/ocaml/idl/schematest.ml @@ -3,7 +3,7 @@ let hash x = Digest.string x |> Digest.to_hex (* BEWARE: if this changes, check that schema has been bumped accordingly in ocaml/idl/datamodel_common.ml, usually schema_minor_vsn *) -let last_known_schema_hash = "186131ad48f40dff30246e8e0c0dbf0a" +let last_known_schema_hash = "a55d5dc70920dcf4ab72ed321497b482" let current_schema_hash : string = let open Datamodel_types in From 5013382c806a85baf75a3b25ce65fa4871766291 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 19 Mar 2024 17:39:00 +0000 Subject: [PATCH 076/149] .git-blame-ignore-revs: ignore another reformatting commit Signed-off-by: Pau Ruiz Safont --- .git-blame-ignore-revs | 1 + 1 file changed, 1 insertion(+) diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 4c2762b5222..d8259ca9cd8 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -29,6 +29,7 @@ b020cf35a1f2c274f95a4118d4596043cba6113f ff39018fd6d91985f9c893a56928771dfe9fa48d cbb9edb17dfd122c591beb14d1275acc39492335 d6ab15362548b8fe270bd14d5153b8d94e1b15c0 +b12cf444edea15da6274975e1b2ca6a7fce2a090 # ocp-indent d018d26d6acd4707a23288b327b49e44f732725e From 85d5c862c2ebe262d8bd293dd6622f202aedf19d Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Thu, 21 Mar 2024 17:51:31 +0000 Subject: [PATCH 077/149] xapi/export: do not miss parameters for export_metadata on logs Signed-off-by: Pau Ruiz Safont --- ocaml/xapi/export.ml | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/ocaml/xapi/export.ml b/ocaml/xapi/export.ml index 54a494ac240..6dd810261b9 100644 --- a/ocaml/xapi/export.ml +++ b/ocaml/xapi/export.ml @@ -607,23 +607,22 @@ let string_of_vm ~__context vm = (** Export a VM's metadata only *) let export_metadata ~__context ~with_snapshot_metadata ~preserve_power_state ~include_vhd_parents ~vms ~excluded_devices s = + let infomsg vm = + info + "VM.export_metadata: VM = %s; with_snapshot_metadata = '%b'; \ + include_vhd_parents = '%b'; preserve_power_state = '%s'; \ + excluded_devices = '%s'" + vm with_snapshot_metadata include_vhd_parents + (string_of_bool preserve_power_state) + (String.concat ", " (List.map Devicetype.to_string excluded_devices)) + in ( match vms with | [] -> failwith "need to specify at least one VM" | [vm] -> - info - "VM.export_metadata: VM = %s; with_snapshot_metadata = '%b'; \ - include_vhd_parents = '%b'; preserve_power_state = '%s" - (string_of_vm ~__context vm) - with_snapshot_metadata include_vhd_parents - (string_of_bool preserve_power_state) + infomsg (string_of_vm ~__context vm) | vms -> - info - "VM.export_metadata: VM = %s; with_snapshot_metadata = '%b'; \ - preserve_power_state = '%s" - (String.concat ", " (List.map (string_of_vm ~__context) vms)) - with_snapshot_metadata - (string_of_bool preserve_power_state) + infomsg (String.concat ", " (List.map (string_of_vm ~__context) vms)) ) ; let _, ova_xml = vm_metadata ~with_snapshot_metadata ~preserve_power_state From 0c6805ffaaa55fdb982901ab6a2d587ffc06225e Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Mon, 25 Mar 2024 15:22:13 +0000 Subject: [PATCH 078/149] xapi/export: set a date when generating tarballs Tooling complains about using 1970 as the date otherwise Signed-off-by: Pau Ruiz Safont --- ocaml/xapi/export.ml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ocaml/xapi/export.ml b/ocaml/xapi/export.ml index 6dd810261b9..c549fb74295 100644 --- a/ocaml/xapi/export.ml +++ b/ocaml/xapi/export.ml @@ -616,6 +616,7 @@ let export_metadata ~__context ~with_snapshot_metadata ~preserve_power_state (string_of_bool preserve_power_state) (String.concat ", " (List.map Devicetype.to_string excluded_devices)) in + let now = Date.now () |> Date.to_unix_time |> Int64.of_float in ( match vms with | [] -> failwith "need to specify at least one VM" @@ -629,7 +630,7 @@ let export_metadata ~__context ~with_snapshot_metadata ~preserve_power_state ~include_vhd_parents ~__context ~vms ~excluded_devices in let hdr = - Tar.Header.make Xapi_globs.ova_xml_filename + Tar.Header.make ~mod_time:now Xapi_globs.ova_xml_filename (Int64.of_int @@ String.length ova_xml) in Tar_helpers.write_block hdr (fun s -> Unixext.really_write_string s ova_xml) s ; @@ -637,6 +638,7 @@ let export_metadata ~__context ~with_snapshot_metadata ~preserve_power_state let export refresh_session __context rpc session_id s vm_ref preserve_power_state = + let now = Date.now () |> Date.to_unix_time |> Int64.of_float in info "VM.export: VM = %s; preserve_power_state = '%s'" (string_of_vm ~__context vm_ref) (string_of_bool preserve_power_state) ; @@ -646,7 +648,7 @@ let export refresh_session __context rpc session_id s vm_ref in debug "Outputting ova.xml" ; let hdr = - Tar.Header.make Xapi_globs.ova_xml_filename + Tar.Header.make ~mod_time:now Xapi_globs.ova_xml_filename (Int64.of_int @@ String.length ova_xml) in Tar_helpers.write_block hdr (fun s -> Unixext.really_write_string s ova_xml) s ; From e2eb7b24f405698954aea52d6e0bf8c17ac814a5 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Sun, 7 Apr 2024 20:53:28 +0100 Subject: [PATCH 079/149] Detect automatically whether we are on cygwin. Signed-off-by: Konstantina Chremmou --- ocaml/sdk-gen/c/README.dist | 1 - ocaml/sdk-gen/c/templates/Makefile.mustache | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ocaml/sdk-gen/c/README.dist b/ocaml/sdk-gen/c/README.dist index dfe92390216..e5fb8622069 100644 --- a/ocaml/sdk-gen/c/README.dist +++ b/ocaml/sdk-gen/c/README.dist @@ -58,4 +58,3 @@ Compiling from source --------------------- To build, simply type "make" in the libxenserver/src directory. -To build on Windows with cygwin type "make CYGWIN=1". diff --git a/ocaml/sdk-gen/c/templates/Makefile.mustache b/ocaml/sdk-gen/c/templates/Makefile.mustache index 384ffcb174d..0718072223a 100644 --- a/ocaml/sdk-gen/c/templates/Makefile.mustache +++ b/ocaml/sdk-gen/c/templates/Makefile.mustache @@ -29,7 +29,9 @@ DESTDIR=/usr/local -ifeq ($(CYGWIN), 1) +UNAME_S := $(shell uname -s) + +ifeq ($(findstring CYGWIN,$(UNAME_S)),CYGWIN) CYGWIN_LIBXML = -L/bin -lxml2-2 POS_FLAG = -U__STRICT_ANSI__ else @@ -80,7 +82,7 @@ install: build $(INSTALL_DATA) libxenserver.so.{{API_VERSION_MAJOR}}.{{API_VERSION_MINOR}} $(DESTDIR)/lib ln -sf libxenserver.so.{{API_VERSION_MAJOR}}.{{API_VERSION_MINOR}} $(DESTDIR)/lib/libxenserver.so.{{API_VERSION_MAJOR}} ln -sf libxenserver.so.{{API_VERSION_MAJOR}} $(DESTDIR)/lib/libxenserver.so -ifeq ($(CYGWIN), 1) +ifeq ($(findstring CYGWIN,$(UNAME_S)),CYGWIN) ln -sf libxenserver.so $(DESTDIR)/lib/libxenserver.dll endif $(INSTALL_DATA) libxenserver.a $(DESTDIR)/lib From ade81daaaad136e1a996f80d0ada739d5e85aa0c Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Thu, 4 Apr 2024 13:23:17 +0100 Subject: [PATCH 080/149] Use templates to generate all the C files. CA-387885 (do not call internal headers from the public ones). Signed-off-by: Konstantina Chremmou --- ocaml/idl/datamodel.ml | 2 +- ocaml/idl/datamodel_observer.ml | 4 +- ocaml/sdk-gen/c/gen_c_binding.ml | 1304 ++++++----------- ocaml/sdk-gen/c/templates/Makefile.mustache | 1 + ocaml/sdk-gen/c/templates/class.c.mustache | 192 +++ ocaml/sdk-gen/c/templates/class.h.mustache | 179 +++ .../sdk-gen/c/templates/class_decl.h.mustache | 47 + ocaml/sdk-gen/c/templates/map.c.mustache | 92 ++ ocaml/sdk-gen/c/templates/map.h.mustache | 68 + ocaml/sdk-gen/c/templates/xen_all.h.mustache | 4 +- .../c/templates/xen_api_failure.c.mustache | 58 + .../c/templates/xen_api_failure.h.mustache | 66 + .../c/templates/xen_api_version.c.mustache | 2 + .../c/templates/xen_api_version.h.mustache | 2 + ocaml/sdk-gen/c/templates/xen_enum.c.mustache | 1 + ocaml/sdk-gen/c/templates/xen_enum.h.mustache | 11 +- .../c/templates/xen_enum_internal.h.mustache | 6 +- .../xen_enum_map_internal.h.mustache | 39 + .../sdk-gen/c/templates/xen_internal.mustache | 2 + 19 files changed, 1208 insertions(+), 872 deletions(-) create mode 100644 ocaml/sdk-gen/c/templates/class.c.mustache create mode 100644 ocaml/sdk-gen/c/templates/class.h.mustache create mode 100644 ocaml/sdk-gen/c/templates/class_decl.h.mustache create mode 100644 ocaml/sdk-gen/c/templates/map.c.mustache create mode 100644 ocaml/sdk-gen/c/templates/map.h.mustache create mode 100644 ocaml/sdk-gen/c/templates/xen_api_failure.c.mustache create mode 100644 ocaml/sdk-gen/c/templates/xen_api_failure.h.mustache create mode 100644 ocaml/sdk-gen/c/templates/xen_enum_map_internal.h.mustache diff --git a/ocaml/idl/datamodel.ml b/ocaml/idl/datamodel.ml index a9f219e91af..11442e2494e 100644 --- a/ocaml/idl/datamodel.ml +++ b/ocaml/idl/datamodel.ml @@ -6070,7 +6070,7 @@ module Event = struct ~doc: "Blocking call which returns a (possibly empty) batch of events. This \ method is only recommended for legacy use. New development should use \ - event.from which supercedes this method." + event.from which supersedes this method." ~custom_marshaller:true ~flags:[`Session] ~result:(Set (Record _event), "A set of events") ~errs:[Api_errors.session_not_registered; Api_errors.events_lost] diff --git a/ocaml/idl/datamodel_observer.ml b/ocaml/idl/datamodel_observer.ml index bbda9021898..1d80d030a62 100644 --- a/ocaml/idl/datamodel_observer.ml +++ b/ocaml/idl/datamodel_observer.ml @@ -95,7 +95,7 @@ let set_components = call ~name:"set_components" ~in_oss_since:None ~lifecycle:[] ~doc: "Set the components on which the observer will broadcast to. i.e. xapi, \ - xenopsd, networkd, etc" + xenopsd, networkd, etc." ~params: [ (Ref _observer, "self", "The observer") @@ -106,7 +106,7 @@ let set_components = let t = create_obj ~name:_observer ~descr: - "Describes a observer which will control observability activity in the \ + "Describes an observer which will control observability activity in the \ Toolstack" ~doccomments:[] ~gen_constructor_destructor:true ~gen_events:true ~in_db:true ~lifecycle:[] ~persist:PersistEverything ~in_oss_since:None diff --git a/ocaml/sdk-gen/c/gen_c_binding.ml b/ocaml/sdk-gen/c/gen_c_binding.ml index 2302fc6cf99..757046ac336 100644 --- a/ocaml/sdk-gen/c/gen_c_binding.ml +++ b/ocaml/sdk-gen/c/gen_c_binding.ml @@ -2,7 +2,7 @@ * Copyright (c) Cloud Software Group, Inc. *) -(* Generator of C bindings from the datamodel *) +(* Generator of the C SDK from the datamodel *) open Printf open Datamodel_types @@ -55,31 +55,23 @@ let enum_maps = ref TypeSet.empty let all_headers = ref [] -let joined sep f l = - let r = List.map f l in - String.concat sep (List.filter (fun x -> String.compare x "" != 0) r) +let rec is_last x list = + match list with + | [] -> + false + | hd :: [] -> + if hd = x then true else false + | hd :: tl -> + if hd = x then false else is_last x tl let rec main () = - let include_dir = Filename.concat destdir "include" in - let src_dir = Filename.concat destdir "src" in - - gen_failure_h () ; - gen_failure_c () ; - let filtered_classes = List.filter (fun x -> not (List.mem x.name ["session"; "debug"; "data_source"])) classes in - List.iter - (fun x -> - ( gen_class write_predecl predecl_filename x include_dir ; - gen_class write_decl decl_filename x include_dir ; - gen_class write_impl impl_filename x - ) - src_dir - ) - filtered_classes ; + List.iter gen_decl filtered_classes ; + List.iter gen_impl filtered_classes ; all_headers := List.map (fun x -> x.name) filtered_classes ; @@ -89,11 +81,12 @@ let rec main () = maps := TypeSet.add (Map (Int, Int)) !maps ; maps := TypeSet.add (Map (String, Set String)) !maps ; maps := TypeSet.add (Map (String, Map (String, String))) !maps ; - TypeSet.iter (gen_map write_map_decl decl_filename include_dir) !maps ; - TypeSet.iter (gen_map write_map_impl impl_filename src_dir) !maps ; + + TypeSet.iter (function Map (l, r) -> render_map_decl l r | _ -> ()) !maps ; + TypeSet.iter (function Map (l, r) -> render_map_impl l r | _ -> ()) !maps ; TypeSet.iter - (gen_map write_enum_map_internal_decl internal_decl_filename include_dir) + (function Map (l, r) -> render_enum_map l r | _ -> ()) !enum_maps ; let class_records = @@ -118,7 +111,10 @@ let rec main () = json1 templates_dir destdir ; let sorted_headers = - List.sort String.compare (List.map decl_filename !all_headers) + !all_headers + |> List.filter (fun x -> not (Astring.String.is_suffix ~affix:"internal" x)) + |> List.map String.lowercase_ascii + |> List.sort String.compare in let json2 = `O @@ -132,295 +128,305 @@ let rec main () = ("xen_all.h.mustache", "include/xen/api/xen_all.h") json2 templates_dir destdir -and gen_class f g clas targetdir = - let out_chan = open_out (Filename.concat targetdir (g clas.name)) in - Fun.protect (fun () -> f clas out_chan) ~finally:(fun () -> close_out out_chan) - -and gen_map f g targetdir = function - | Map (l, r) -> - let name = mapname l r in - if not (List.mem name !all_headers) then - all_headers := name :: !all_headers ; - let out_chan = open_out (Filename.concat targetdir (g name)) in - Fun.protect - (fun () -> f name l r out_chan) - ~finally:(fun () -> close_out out_chan) - | _ -> - assert false - -and write_predecl {name= classname; _} out_chan = - let print format = fprintf out_chan format in - let protect = protector (classname ^ "_decl") in - let tn = typename classname in - let record_tn = record_typename classname in - let record_opt_tn = record_opt_typename classname in - - print_h_header out_chan protect ; - - if classname <> "event" then ( - print "typedef void *%s;\n\n" tn ; - print "%s\n" (predecl_set tn) - ) ; - print "%s\n" (predecl record_tn) ; - print "%s\n" (predecl_set record_tn) ; - if classname <> "event" then ( - print "%s\n" (predecl record_opt_tn) ; - print "%s\n" (predecl_set record_opt_tn) - ) ; - print_h_footer out_chan +and gen_decl cls = + let headers = ref (StringSet.add (cls.name ^ "_decl") StringSet.empty) in + let rec get_needed = function + | Field fr -> + find_needed headers fr.ty + | Namespace (_, cs) -> + List.iter get_needed cs + in + List.iter get_needed cls.contents ; -and write_decl {name= classname; contents; description; messages; _} out_chan = - let print format = fprintf out_chan format in - let protect = protector classname in - let tn = typename classname in - let record_tn = record_typename classname in - let record_opt_tn = record_opt_typename classname in - let class_has_refs = true (* !!! *) in - let needed = ref (StringSet.add (classname ^ "_decl") StringSet.empty) in - let record = decl_record needed tn record_tn contents in - let record_opt = decl_record_opt tn record_tn record_opt_tn in - let message_decls = - decl_messages needed classname - (List.filter - (fun x -> not (classname = "event" && x.msg_name = "from")) - messages - ) + let asyncParams x = + if x.msg_async then + { + param_type= Ref "task" + ; param_name= "*result" + ; param_doc= "" + ; param_release= x.msg_release + ; param_default= None + } + :: x.msg_params + else + x.msg_params in - let full_stop = - if Astring.String.is_suffix ~affix:"." description then "" else "." + let syncParams x = + match x.msg_result with + | Some res -> + { + param_type= fst res + ; param_name= "*result" + ; param_doc= "" + ; param_release= x.msg_release + ; param_default= None + } + :: x.msg_params + | None -> + x.msg_params + in + let paramJson x = + `O + [ + ("param_name", `String (paramname x.param_name)) + ; ("param_type", `String (c_type_of_ty headers false x.param_type)) + ] in + let json = + `O + [ + ("class_upper", `String (String.uppercase_ascii cls.name)) + ; ("class_lower", `String (String.lowercase_ascii cls.name)) + ; ("class_doc", `String (Helper.comment false (full_class_doc cls))) + ; ("is_event", `Bool (cls.name = "event")) + ; ( "headers" + , `A + (List.map + (fun x -> `O [("header", `String x)]) + ("common" :: StringSet.elements !headers + |> List.map String.lowercase_ascii + |> List.sort String.compare + |> List.filter (fun x -> + not (Astring.String.is_suffix ~affix:"internal" x) + ) + ) + ) + ) + ; ( "fields" + , `A + (cls + |> Datamodel_utils.fields_of_obj + |> List.map (fun field -> + `O + [ + ( "field_name_lower" + , `String (fieldname (String.concat "_" field.full_name)) + ) + ; ( "field_type" + , `String (c_type_of_ty headers true field.ty) + ) + ] + ) + ) + ) + ; ( "messages" + , `A + (cls.messages + |> List.filter (fun x -> + not (cls.name = "event" && x.msg_name = "from") + ) + |> List.map (fun x -> + `O + [ + ( "msg_name_lower" + , `String (String.lowercase_ascii x.msg_name) + ) + ; ( "msg_doc" + , `String (Helper.comment true (full_msg_doc x)) + ) + ; ("is_async", `Bool x.msg_async) + ; ("sync_params", `A (List.map paramJson (syncParams x))) + ; ("async_params", `A (List.map paramJson (asyncParams x))) + ] + ) + ) + ) + ] + in + render_file + ( "class_decl.h.mustache" + , sprintf "include/xen/api/xen_%s_decl.h" (String.lowercase_ascii cls.name) + ) + json templates_dir destdir ; + render_file + ( "class.h.mustache" + , sprintf "include/xen/api/xen_%s.h" (String.lowercase_ascii cls.name) + ) + json templates_dir destdir - let rec get_needed x = - match x with +and gen_impl cls = + let headers = ref StringSet.empty in + let rec get_needed = function | Field fr -> - find_needed'' needed fr.ty + find_needed headers fr.ty | Namespace (_, cs) -> List.iter get_needed cs in - List.iter get_needed contents ; - - print_h_header out_chan protect ; - print "%s\n" (hash_includes !needed) ; - - print "\n\n%s\n\n\n" - (Helper.comment false - (sprintf "The %s class.\n\n%s%s" classname description full_stop) - ) ; + List.iter get_needed cls.contents ; - if classname <> "event" then ( - print "%s\n\n" - (decl_free tn (String.lowercase_ascii classname) false "handle") ; - print "%s\n" (decl_set tn false) - ) ; - print "%s\n" record ; - if classname <> "event" then - print "%s\n" record_opt ; - print "%s\n\n" (decl_set record_tn class_has_refs) ; - if classname <> "event" then - print "%s\n\n" (decl_set record_opt_tn true) ; - print "%s\n" message_decls ; - print_h_footer out_chan - -and predecl_set tn = predecl (tn ^ "_set") - -and predecl tn = sprintf "struct %s;" tn - -and decl_set tn referenced = - let alloc_com = - Helper.comment true (sprintf "Allocate a %s_set of the given size." tn) - in - - sprintf - "\n\ - typedef struct %s_set\n\ - {\n\ - \ size_t size;\n\ - \ %s *contents[];\n\ - } %s_set;\n\n\ - %s\n\ - extern %s_set *\n\ - %s_set_alloc(size_t size);\n\n\ - %s\n" - tn tn tn alloc_com tn tn - (decl_free (sprintf "%s_set" tn) "*set" referenced "set") + List.iter + (fun x -> + List.iter (fun p -> find_needed headers p.param_type) x.msg_params ; + match x.msg_result with + | Some res -> + find_needed headers (fst res) + | None -> + () + ) + cls.messages ; -and decl_free tn cn referenced thing = - let com = - Helper.comment true - (sprintf - "Free the given %s%s. The given %s must have been allocated by this \ - library." - tn - (if referenced then ", and all referenced values" else "") - thing - ) + let allFields = cls |> Datamodel_utils.fields_of_obj in + let result_type message = + match message.msg_result with + | Some res -> + abstract_type false (fst res) + | None -> + "" in - - sprintf "%s\nextern void\n%s_free(%s %s);" com tn tn cn - -and decl_record needed tn record_tn contents = - sprintf - "\n\ - typedef struct %s\n\ - {\n\ - %s %s\n\ - } %s;\n\n\ - %s\n\ - extern %s *\n\ - %s_alloc(void);\n\n\ - %s\n" - record_tn - (if tn <> "xen_event" then sprintf " %s handle;\n" tn else "") - (record_fields contents needed) - record_tn - (Helper.comment true (sprintf "Allocate a %s." record_tn)) - record_tn record_tn - (decl_free record_tn "*record" true "record") - -and decl_record_opt tn record_tn record_opt_tn = - sprintf - "\n\ - typedef struct %s\n\ - {\n\ - \ bool is_record;\n\ - \ union\n\ - \ {\n\ - \ %s handle;\n\ - \ %s *record;\n\ - \ } u;\n\ - } %s;\n\n\ - %s\n\ - extern %s *\n\ - %s_alloc(void);\n\n\ - %s\n" - record_opt_tn tn record_tn record_opt_tn - (Helper.comment true (sprintf "Allocate a %s." record_opt_tn)) - record_opt_tn record_opt_tn - (decl_free record_opt_tn "*record_opt" true "record_opt") - -and record_fields contents needed = - joined "\n " (record_field needed "") contents - -and record_field needed prefix content = - match content with - | Field fr -> - sprintf "%s%s%s;" - (c_type_of_ty needed true fr.ty) - prefix (fieldname fr.field_name) - | Namespace (p, c) -> - joined "\n " (record_field needed (prefix ^ fieldname p ^ "_")) c - -and decl_messages needed classname messages = - joined "\n\n" (decl_message needed classname) messages - -and decl_message needed classname message = - let message_sig = message_signature needed classname message in - let messageAsyncVersion = decl_message_async needed classname message in - sprintf "%s\n%sextern %s;\n%s" - (get_message_comment message) - (get_deprecated_message message) - message_sig messageAsyncVersion - -and decl_message_async needed classname message = - if message.msg_async then ( - let messageSigAsync = message_signature_async needed classname message in - needed := StringSet.add "task_decl" !needed ; - sprintf "\n%s\n%sextern %s;\n" - (get_message_comment message) - (get_deprecated_message message) - messageSigAsync - ) else - "" - -and get_message_comment message = - let full_stop = - if Astring.String.is_suffix ~affix:"." message.msg_doc then "" else "." + let init_result message = + match message.msg_result with + | Some res -> ( + match fst res with + | SecretString | String | Ref _ | Set _ | Map _ | Record _ -> + true + | _ -> + false + ) + | None -> + false in - let minimum_allowed_role = get_minimum_allowed_role message in - let content = - sprintf "%s%s\nMinimum allowed role: %s." message.msg_doc full_stop - minimum_allowed_role + let is_result_record message = + match message.msg_result with + | Some res -> ( + match fst res with Record _ -> true | _ -> false + ) + | None -> + false in - Helper.comment true content - -and impl_messages needed classname messages = - joined "\n\n" (impl_message needed classname) messages - -and impl_message needed classname message = - let message_sig = message_signature needed classname message in - let param_count = List.length message.msg_params in - - let param_decl, param_call = - if param_count = 0 then - ("", "NULL") + let asyncParams x = + if x.msg_async then + { + param_type= Ref "task" + ; param_name= "*result" + ; param_doc= "" + ; param_release= x.msg_release + ; param_default= None + } + :: x.msg_params else - let param_pieces = abstract_params message.msg_params in - - ( sprintf - " abstract_value param_values[] =\n\ - \ {\n\ - \ %s\n\ - \ };\n" - param_pieces - , "param_values" - ) + x.msg_params in - - let result_bits = - match message.msg_result with + let syncParams x = + match x.msg_result with | Some res -> - abstract_result_handling classname message.msg_name param_count res + { + param_type= fst res + ; param_name= "*result" + ; param_doc= "" + ; param_release= x.msg_release + ; param_default= None + } + :: x.msg_params | None -> - sprintf - " xen_call_(session, \"%s.%s\", %s, %d, NULL, NULL);\n\ - \ return session->ok;\n" - classname message.msg_name - (if param_count = 0 then "NULL" else param_call) - param_count + x.msg_params in - - let messageAsyncImpl = impl_message_async needed classname message in - sprintf "%s%s\n{\n%s\n%s}\n%s" - (get_deprecated_message message) - message_sig param_decl result_bits messageAsyncImpl - -and impl_message_async needed classname message = - if message.msg_async then - let messageSigAsync = message_signature_async needed classname message in - let param_count = List.length message.msg_params in - - let param_decl, _ = - if param_count = 0 then - ("", "NULL") - else - let param_pieces = abstract_params message.msg_params in - - ( sprintf - " abstract_value param_values[] =\n\ - \ {\n\ - \ %s\n\ - \ };\n" - param_pieces - , "param_values" - ) + let messageJson msg = + let paramJson p = + `O + [ + ("param_name", `String (paramname p.param_name)) + ; ("param_type", `String (c_type_of_ty headers false p.param_type)) + ; ("abstract_param_type", `String (abstract_type false p.param_type)) + ; ("abstract_member", `String (abstract_member p.param_type)) + ; ( "abstract_member_conv" + , `String (abstract_param_conv p.param_name p.param_type) + ) + ; ("is_last", `Bool (is_last p msg.msg_params)) + ] in + `O + [ + ("msg_name_lower", `String (String.lowercase_ascii msg.msg_name)) + ; ("msg_name", `String msg.msg_name) + ; ("msg_doc", `String (Helper.comment true (full_msg_doc msg))) + ; ("is_async", `Bool msg.msg_async) + ; ("sync_params", `A (List.map paramJson (syncParams msg))) + ; ("async_params", `A (List.map paramJson (asyncParams msg))) + ; ("msg_params", `A (List.map paramJson msg.msg_params)) + ; ("abstract_result_type", `String (result_type msg)) + ; ("has_params", `Bool (List.length msg.msg_params <> 0)) + ; ("param_count", `String (string_of_int (List.length msg.msg_params))) + ; ("has_result", `Bool (String.compare (result_type msg) "" <> 0)) + ; ("init_result", `Bool (init_result msg)) + ; ("is_result_record", `Bool (is_result_record msg)) + ] + in + let fieldJson field = + let fullName = String.concat "_" field.full_name in + let freeing = free_impl ("record->" ^ fieldname fullName) true field.ty in + `O + [ + ("field_name_lower", `String (fieldname fullName)) + ; ("field_name", `String fullName) + ; ("abstract_field_type", `String (abstract_type true field.ty)) + ; ("can_free", `Bool (freeing <> "")) + ; ("free_record_field", `String freeing) + ; ("is_last", `Bool (is_last field allFields)) + ] + in + let json = + `O + [ + ("class_name", `String cls.name) + ; ("class_lower", `String (String.lowercase_ascii cls.name)) + ; ("is_event", `Bool (cls.name = "event")) + ; ( "has_all_records" + , `Bool + (List.exists (fun x -> x.msg_name = "get_all_records") cls.messages) + ) + ; ( "headers" + , `A + (List.map + (fun x -> `O [("header", `String x)]) + (["common"; String.lowercase_ascii cls.name] + |> List.sort String.compare + ) + ) + ) + ; ( "internal_headers" + , `A + (List.map + (fun x -> `O [("header", `String x)]) + ("internal" :: StringSet.elements !headers + |> List.map String.lowercase_ascii + |> List.sort String.compare + |> List.filter (fun x -> + Astring.String.is_suffix ~affix:"internal" x + ) + ) + ) + ) + ; ("fields", `A (allFields |> List.map fieldJson)) + ; ( "messages" + , `A + (cls.messages + |> List.filter (fun x -> + not (cls.name = "event" && x.msg_name = "from") + ) + |> List.map messageJson + ) + ) + ] + in + render_file + ( "class.c.mustache" + , sprintf "src/xen_%s.c" (String.lowercase_ascii cls.name) + ) + json templates_dir destdir - let result_bits = - abstract_result_handling_async classname message.msg_name param_count - in - sprintf "\n%s%s\n{\n%s\n%s}" - (get_deprecated_message message) - messageSigAsync param_decl result_bits - else - "" +and full_stop x = if Astring.String.is_suffix ~affix:"." x then "" else "." -and abstract_params params = joined ",\n " abstract_param params +and full_class_doc cls = + let intro = sprintf "The %s class.\n\n" cls.name in + intro ^ cls.description ^ full_stop cls.description -and abstract_param p = - let ab_typ = abstract_type false p.param_type in - sprintf "{ .type = &%s,\n .u.%s_val = %s }" ab_typ - (abstract_member p.param_type) - (abstract_param_conv p.param_name p.param_type) +and full_msg_doc message = + let role = + sprintf "\nMinimum allowed role: %s." (get_minimum_allowed_role message) + in + let deprecated = get_deprecated_info_message message in + let deprecated = if deprecated = "" then "" else "\n" ^ deprecated in + message.msg_doc ^ full_stop message.msg_doc ^ role ^ deprecated and abstract_param_conv name = function | Set _ | Map _ -> @@ -443,9 +449,7 @@ and abstract_member = function "bool" | DateTime -> "datetime" - | Set _ -> - "set" - | Map _ -> + | Set _ | Map _ -> "set" | Record _ -> "struct" @@ -453,81 +457,6 @@ and abstract_member = function eprintf "%s" (Types.to_string x) ; assert false -and abstract_result_handling classname msg_name param_count = function - | typ, _ -> ( - let call = - if param_count = 0 then - sprintf - "xen_call_(session, \"%s.%s\", NULL, 0, &result_type, result);" - classname msg_name - else - sprintf "XEN_CALL_(\"%s.%s\");" classname msg_name - in - - match typ with - | String | Ref _ | Int | Float | Bool | DateTime | Set _ | Map _ -> - sprintf "%s\n\n%s %s\n return session->ok;\n" - (abstract_result_type typ) (initialiser_of_ty typ) call - | Record n -> - let record_tn = record_typename n in - sprintf - " abstract_type result_type = %s_abstract_type_;\n\n\ - %s %s\n\n\ - \ if (session->ok)\n\ - \ {\n\ - \ (*result)->handle = xen_strdup_((*result)->uuid);\n\ - \ }\n\n\ - \ return session->ok;\n" - record_tn - (initialiser_of_ty (Record n)) - call - | Enum (_, _) -> - sprintf "%s\n %s\n return session->ok;\n" - (abstract_result_type typ) call - | x -> - eprintf "%s" (Types.to_string x) ; - assert false - ) - -and abstract_result_handling_async classname msg_name param_count = - let call = - if param_count = 0 then - sprintf - "xen_call_(session, \"Async.%s.%s\", NULL, 0, &result_type, result);" - classname msg_name - else - sprintf "XEN_CALL_(\"Async.%s.%s\");" classname msg_name - in - sprintf - " abstract_type result_type = abstract_type_string;\n\n\ - \ *result = NULL;\n\ - \ %s\n\ - \ return session->ok;\n" - call - -and abstract_record_field classname prefix prefix_caps content = - match content with - | Field fr -> - let fn = fieldname fr.field_name in - sprintf - "{ .key = \"%s%s\",\n\ - \ .type = &%s,\n\ - \ .offset = offsetof(%s, %s%s) }" prefix_caps fr.field_name - (abstract_type true fr.ty) - (record_typename classname) - prefix fn - | Namespace (p, c) -> - joined ",\n " - (abstract_record_field classname - (prefix ^ fieldname p ^ "_") - (prefix_caps ^ p ^ "_") - ) - c - -and abstract_result_type typ = - let ab_typ = abstract_type false typ in - sprintf " abstract_type result_type = %s;" ab_typ - and abstract_type record = function | SecretString | String -> "abstract_type_string" @@ -575,87 +504,6 @@ and abstract_type record = function | Option n -> abstract_type record n -and get_deprecated_message message = - let deprecatedMessage = get_deprecated_info_message message in - if deprecatedMessage = "" then - sprintf "" - else - sprintf "/* " ^ deprecatedMessage ^ " */\n" - -and message_signature needed classname message = - let front = - { - param_type= Ref "session" - ; param_name= "session" - ; param_doc= "" - ; param_release= message.msg_release - ; param_default= None - } - :: - ( match message.msg_result with - | Some res -> - [ - { - param_type= fst res - ; param_name= "*result" - ; param_doc= "" - ; param_release= message.msg_release - ; param_default= None - } - ] - | None -> - [] - ) - in - let params = joined ", " (param needed) (front @ message.msg_params) in - sprintf "bool\n%s(%s)" (messagename classname message.msg_name) params - -and message_signature_async needed classname message = - let sessionParam = - { - param_type= Ref "session" - ; param_name= "session" - ; param_doc= "" - ; param_release= message.msg_release - ; param_default= None - } - in - let taskParam = - { - param_type= Ref "task" - ; param_name= "*result" - ; param_doc= "" - ; param_release= message.msg_release - ; param_default= None - } - in - let params = - joined ", " (param needed) (sessionParam :: taskParam :: message.msg_params) - in - sprintf "bool\n%s(%s)" (messagename_async classname message.msg_name) params - -and param needed p = - let t = p.param_type in - let n = p.param_name in - sprintf "%s%s" (c_type_of_ty needed false t) (paramname n) - -and hash_includes needed = - String.concat "\n" - (List.sort String.compare - (List.filter - (function s -> s <> "") - (List.map hash_include ("common" :: StringSet.elements needed)) - ) - ) - -and hash_include n = - if Astring.String.is_suffix ~affix:"internal" n then - sprintf "#include \"%s\"" (decl_filename n) - else if n = "session" then - "" - else - sprintf "#include <%s>" (decl_filename n) - and replace_dashes x = Astring.String.map (fun y -> match y with '-' -> '_' | _ -> y) x @@ -677,7 +525,9 @@ and render_enum x = `O [ ("enum_value", `String n) - ; ("enum_value_doc", `String c) + ; ( "enum_value_doc" + , `String (Helper.comment true ~indent:4 c) + ) ; ( "enum_value_upper" , `String (replace_dashes (String.uppercase_ascii n)) ) @@ -702,371 +552,159 @@ and render_enum x = | _ -> () -and write_map_decl name l r out_chan = - let print format = fprintf out_chan format in - let tn = typename name in - let protect = protector name in - let needed = ref StringSet.empty in - let alloc_com = - Helper.comment true (sprintf "Allocate a %s of the given size." tn) - in - - print_h_header out_chan protect ; - print - "\n\ - %s%s%s\n\n\n\ - typedef struct %s_contents\n\ - {\n\ - \ %skey;\n\ - \ %sval;\n\ - } %s_contents;\n\n\n\ - typedef struct %s\n\ - {\n\ - \ size_t size;\n\ - \ %s_contents contents[];\n\ - } %s;\n\n\ - %s\n\ - extern %s *\n\ - %s_alloc(size_t size);\n\n\ - %s\n\n" - (hash_include "common") (hash_include_enum l) (hash_include_enum r) tn - (c_type_of_ty needed false l) - (c_type_of_ty needed true r) - tn tn tn tn alloc_com tn tn - (decl_free tn "*map" true "map") ; - print_h_footer out_chan - -and write_map_impl name l r out_chan = - let print format = fprintf out_chan format in - let tn = typename name in - let l_free_impl = free_impl "map->contents[i].key" false l in - let r_free_impl = free_impl "map->contents[i].val" true r in - let needed = ref StringSet.empty in - find_needed'' needed l ; - find_needed'' needed r ; - needed := StringSet.add "internal" !needed ; - needed := StringSet.add name !needed ; - ( match r with - | Set String -> - needed := StringSet.add "string_set" !needed - | _ -> - () - ) ; - - print - "%s\n\n\n\ - %s\n\n\n\ - %s *\n\ - %s_alloc(size_t size)\n\ - {\n\ - \ %s *result = calloc(1, sizeof(%s) +\n\ - \ %s size * sizeof(struct %s_contents));\n\ - \ result->size = size;\n\ - \ return result;\n\ - }\n\n\n\ - void\n\ - %s_free(%s *map)\n\ - {\n" - Licence.bsd_two_clause (hash_includes !needed) tn tn tn tn - (String.make (String.length tn) ' ') - tn tn tn ; - - if String.compare l_free_impl "" != 0 || String.compare r_free_impl "" != 0 - then - print - " if (map == NULL)\n\ - \ {\n\ - \ return;\n\ - \ }\n\n\ - \ size_t n = map->size;\n\ - \ for (size_t i = 0; i < n; i++)\n\ - \ {\n\ - \ %s\n\ - \ %s\n\ - \ }\n\n" - l_free_impl r_free_impl ; - - print " free(map);\n}\n" ; - - match (l, r) with - | Enum (_, _), _ -> - gen_enum_map_abstract_type print l r - | _, Enum (_, _) -> - gen_enum_map_abstract_type print l r - | _ -> - () - -and gen_enum_map_abstract_type print l r = - let tn = mapname l r in - print - "\n\n\ - static const struct_member %s_struct_members[] =\n\ - \ {\n\ - \ { .type = &%s,\n\ - \ .offset = offsetof(xen_%s_contents, key) },\n\ - \ { .type = &%s,\n\ - \ .offset = offsetof(xen_%s_contents, val) },\n\ - \ };\n\n\ - const abstract_type %s_abstract_type_ =\n\ - \ {\n\ - \ .XEN_API_TYPE = MAP,\n\ - \ .struct_size = sizeof(%s_struct_members),\n\ - \ .member_count =\n\ - \ sizeof(%s_struct_members) / sizeof(struct_member),\n\ - \ .members = %s_struct_members\n\ - \ };\n" - tn (abstract_type false l) tn (abstract_type false r) tn tn tn tn tn - -and write_enum_map_internal_decl name l r out_chan = - let print format = fprintf out_chan format in - let protect = protector (sprintf "%s_internal" name) in - - print_h_header out_chan protect ; - print "\nextern const abstract_type %s_abstract_type_;\n\n" (mapname l r) ; - print_h_footer out_chan - -and hash_include_enum = function - | Enum (x, _) -> - "\n" ^ hash_include x - | _ -> - "" - -and gen_failure_h () = - let protect = protector "api_failure" in - let out_chan = - open_out (Filename.concat destdir "include/xen/api/xen_api_failure.h") +and render_enum_map l r = + let x = mapname l r in + let json = + `O + [ + ("map_upper", `String (String.uppercase_ascii x)) + ; ("map_lower", `String (String.lowercase_ascii x)) + ] in - Fun.protect - (fun () -> - print_h_header out_chan protect ; - gen_failure_enum out_chan ; - gen_failure_funcs out_chan ; - print_h_footer out_chan + render_file + ( "xen_enum_map_internal.h.mustache" + , sprintf "include/xen_%s_internal.h" (String.lowercase_ascii x) ) - ~finally:(fun () -> close_out out_chan) - -and gen_failure_enum out_chan = - let print format = fprintf out_chan format in - print "\nenum xen_api_failure\n{\n%s\n};\n\n\n" - (String.concat ",\n\n" (failure_enum_entries ())) - -and failure_enum_entries () = - let r = Hashtbl.fold failure_enum_entry Datamodel.errors [] in - let r = List.sort (fun (x, _) (y, _) -> String.compare y x) r in - let r = - failure_enum_entry "UNDEFINED" - { - err_doc= "Unknown to this version of the bindings." - ; err_params= [] - ; err_name= "UNDEFINED" - } - r + json templates_dir destdir + +and render_map_decl l r = + let headers = ref StringSet.empty in + let add_enum_header = function + | Enum (x, _) -> + headers := StringSet.add x !headers + | _ -> + () + in + add_enum_header l ; + add_enum_header r ; + let x = mapname l r in + let json = + `O + [ + ("key_type_lower", `String (c_type_of_ty headers false l)) + ; ("val_type_lower", `String (c_type_of_ty headers true r)) + ; ("map_upper", `String (String.uppercase_ascii x)) + ; ("map_lower", `String (String.lowercase_ascii x)) + ; ( "headers" + , `A + (List.map + (fun x -> `O [("header", `String x)]) + ("common" :: StringSet.elements !headers + |> List.map String.lowercase_ascii + |> List.sort String.compare + |> List.filter (fun x -> + not (Astring.String.is_suffix ~affix:"internal" x) + ) + ) + ) + ) + ] in - List.map (fun (_, y) -> y) (List.rev r) - -and failure_enum_entry name err acc = - ( name - , sprintf "%s\n %s" - (Helper.comment true ~indent:4 err.Datamodel_types.err_doc) - (failure_enum name) - ) - :: acc - -and gen_failure_funcs out_chan = - let print format = fprintf out_chan format in - print - "%s\n\ - extern const char *\n\ - xen_api_failure_to_string(enum xen_api_failure val);\n\n\n\ - %s\n\ - extern enum xen_api_failure\n\ - xen_api_failure_from_string(const char *str);\n\n" - (Helper.comment true - "Return the name corresponding to the given code. This string must not \ - be modified or freed." - ) - (Helper.comment true - "Return the correct code for the given string, or UNDEFINED if the \ - given string does not match a known code." - ) - -and gen_failure_c () = - let out_chan = open_out (Filename.concat destdir "src/xen_api_failure.c") in - let print format = fprintf out_chan format in - Fun.protect - (fun () -> - print - "%s\n\n\ - #include \"xen_internal.h\"\n\ - #include \n\n\n\ - /*\n\ - \ * Maintain this in the same order as the enum declaration!\n\ - \ */\n\ - static const char *lookup_table[] =\n\ - {\n\ - \ %s\n\ - };\n\n\n\ - const char *\n\ - xen_api_failure_to_string(enum xen_api_failure val)\n\ - {\n\ - \ return lookup_table[val];\n\ - }\n\n\n\ - extern enum xen_api_failure\n\ - xen_api_failure_from_string(const char *str)\n\ - {\n\ - \ return ENUM_LOOKUP(str, lookup_table);\n\ - }\n\n\n" - Licence.bsd_two_clause - (String.concat ",\n " (failure_lookup_entries ())) + if not (List.mem x !all_headers) then all_headers := x :: !all_headers ; + render_file + ( "map.h.mustache" + , sprintf "include/xen/api/xen_%s.h" (String.lowercase_ascii x) ) - ~finally:(fun () -> close_out out_chan) + json templates_dir destdir -and failure_lookup_entries () = - List.sort String.compare - (Hashtbl.fold failure_lookup_entry Datamodel.errors []) +and render_map_impl l r = + let x = mapname l r in + let headers = ref StringSet.empty in + headers := StringSet.add x !headers ; + find_needed headers l ; + find_needed headers r ; -and failure_lookup_entry name _ acc = sprintf "\"%s\"" name :: acc - -and failure_enum name = "XEN_API_FAILURE_" ^ String.uppercase_ascii name - -and write_impl {name= classname; contents; messages; _} out_chan = - let is_event = classname = "event" in - let print format = fprintf out_chan format in - let needed = ref StringSet.empty in - let tn = typename classname in - let record_tn = record_typename classname in - let record_opt_tn = record_opt_typename classname in - let msgs = - impl_messages needed classname - (List.filter - (fun x -> not (classname = "event" && x.msg_name = "from")) - messages - ) - in - let record_free_handle = - if classname = "event" then "" else " free(record->handle);\n" - in - let record_free_impls = - joined "\n " (record_free_impl "record->") contents - in - let filtered_record_fields = - let not_obj_uuid x = - match x with Field r when r.field_name = "obj_uuid" -> false | _ -> true - in - if is_event then List.filter not_obj_uuid contents else contents - in - let record_fields = - joined ",\n " - (abstract_record_field classname "" "") - filtered_record_fields + let l_free_impl = free_impl "map->contents[i].key" false l in + let r_free_impl = free_impl "map->contents[i].val" true r in + let is_enum_map = + match (l, r) with Enum (_, _), _ | _, Enum (_, _) -> true | _ -> false in - let needed = ref StringSet.empty in - find_needed needed messages ; - needed := StringSet.add "internal" !needed ; - needed := StringSet.add classname !needed ; - - let getAllRecordsExists = - List.exists (fun x -> x.msg_name = "get_all_records") messages + let json = + `O + [ + ("abstract_type_key", `String (abstract_type false l)) + ; ("abstract_type_val", `String (abstract_type false r)) + ; ("map_upper", `String (String.uppercase_ascii x)) + ; ("map_lower", `String (String.lowercase_ascii x)) + ; ( "headers" + , `A + (List.map + (fun x -> `O [("header", `String x)]) + ("common" :: StringSet.elements !headers + |> List.map String.lowercase_ascii + |> List.sort String.compare + |> List.filter (fun x -> + not (Astring.String.is_suffix ~affix:"internal" x) + ) + ) + ) + ) + ; ( "internal_headers" + , `A + (List.map + (fun x -> `O [("header", `String x)]) + ("internal" :: StringSet.elements !headers + |> List.map String.lowercase_ascii + |> List.sort String.compare + |> List.filter (fun x -> + Astring.String.is_suffix ~affix:"internal" x + ) + ) + ) + ) + ; ("can_free_key", `Bool (String.compare l_free_impl "" != 0)) + ; ("can_free_val", `Bool (String.compare r_free_impl "" != 0)) + ; ( "can_free" + , `Bool + (String.compare l_free_impl "" != 0 + || String.compare r_free_impl "" != 0 + ) + ) + ; ("free_key", `String l_free_impl) + ; ("free_val", `String r_free_impl) + ; ("enum_map", `Bool is_enum_map) + ] in - let mappingName = sprintf "%s_%s" tn record_tn in - - let free_block = - String.concat "\n" - (( if is_event then - [] - else - [sprintf "XEN_FREE(%s)" tn; sprintf "XEN_SET_ALLOC_FREE(%s)" tn] - ) - @ [ - sprintf "XEN_ALLOC(%s)" record_tn - ; sprintf "XEN_SET_ALLOC_FREE(%s)" record_tn - ] - @ - if is_event then - [] - else - [ - sprintf "XEN_ALLOC(%s)" record_opt_tn - ; sprintf "XEN_RECORD_OPT_FREE(%s)" tn - ; sprintf "XEN_SET_ALLOC_FREE(%s)" record_opt_tn - ] + if not (List.mem x !all_headers) then all_headers := x :: !all_headers ; + render_file + ("map.c.mustache", sprintf "src/xen_%s.c" (String.lowercase_ascii x)) + json templates_dir destdir + +and gen_failure () = + let errors = + Hashtbl.fold + (fun _ x acc -> + (x.Datamodel_types.err_name, x.Datamodel_types.err_doc) :: acc ) + Datamodel.errors [] in + let errors = List.sort (fun (x, _) (y, _) -> String.compare x y) errors in + let json = + `O + [ + ( "api_errors" + , `A + (List.map + (fun (x, y) -> + `O + [ + ("api_error", `String (String.uppercase_ascii x)) + ; ("api_error_doc", `String (Helper.comment true ~indent:4 y)) + ] + ) + errors + ) + ) + ] + in + render_file + ("xen_api_failure.h.mustache", "include/xen/api/xen_api_failure.h") + json templates_dir destdir ; + render_file + ("xen_api_failure.c.mustache", "src/xen_api_failure.c") + json templates_dir destdir - print "%s\n\n\n#include \n#include \n\n%s\n\n\n%s\n\n\n" - Licence.bsd_two_clause (hash_includes !needed) free_block ; - - print - "static const struct_member %s_struct_members[] =\n\ - \ {\n\ - \ %s\n\ - \ };\n\n\ - const abstract_type %s_abstract_type_ =\n\ - \ {\n\ - \ .XEN_API_TYPE = STRUCT,\n\ - \ .struct_size = sizeof(%s),\n\ - \ .member_count =\n\ - \ sizeof(%s_struct_members) / sizeof(struct_member),\n\ - \ .members = %s_struct_members\n\ - \ };\n\n\n" - record_tn record_fields record_tn record_tn record_tn record_tn ; - - print - "const abstract_type %s_set_abstract_type_ =\n\ - \ {\n\ - \ .XEN_API_TYPE = SET,\n\ - \ .child = &%s_abstract_type_\n\ - \ };\n\n\n" - record_tn record_tn ; - - if getAllRecordsExists then - print - "static const struct struct_member %s_members[] =\n\ - {\n\ - \ {\n\ - \ .type = &abstract_type_string,\n\ - \ .offset = offsetof(%s_map_contents, key)\n\ - \ },\n\ - \ {\n\ - \ .type = &%s_abstract_type_,\n\ - \ .offset = offsetof(%s_map_contents, val)\n\ - \ }\n\ - };\n\n\ - const abstract_type abstract_type_string_%s_map =\n\ - {\n\ - \ .XEN_API_TYPE = MAP,\n\ - \ .struct_size = sizeof(%s_map_contents),\n\ - \ .members = %s_members\n\ - };\n\n\n" - mappingName mappingName record_tn mappingName record_tn mappingName - mappingName ; - - print - "void\n\ - %s_free(%s *record)\n\ - {\n\ - \ if (record == NULL)\n\ - \ {\n\ - \ return;\n\ - \ }\n\ - %s %s\n\ - \ free(record);\n\ - }\n\n\n" - record_tn record_tn record_free_handle record_free_impls ; - - print "%s\n" msgs - -and find_needed needed messages = List.iter (find_needed' needed) messages - -and find_needed' needed message = - List.iter (fun p -> find_needed'' needed p.param_type) message.msg_params ; - match message.msg_result with - | Some (x, _) -> - find_needed'' needed x - | None -> - () - -and find_needed'' needed = function +and find_needed needed = function | SecretString | String | Int | Float | Bool | DateTime -> () | Enum (n, _) -> @@ -1091,13 +729,7 @@ and find_needed'' needed = function | Record n -> needed := StringSet.add n !needed | Option x -> - find_needed'' needed x - -and record_free_impl prefix = function - | Field fr -> - free_impl (prefix ^ fieldname fr.field_name) true fr.ty - | Namespace (p, c) -> - joined "\n " (record_free_impl (prefix ^ fieldname p ^ "_")) c + find_needed needed x and free_impl val_name record = function | SecretString | String -> @@ -1167,7 +799,7 @@ and c_type_of_ty needed record = function | Enum (name, _) as x -> needed := StringSet.add name !needed ; enums := TypeSet.add x !enums ; - c_type_of_enum name + sprintf "enum %s " (typename name) | Set (Ref name) -> needed := StringSet.add (name ^ "_decl") !needed ; if record then @@ -1220,23 +852,13 @@ and c_type_of_ty needed record = function | Option (Enum (name, _) as x) -> needed := StringSet.add name !needed ; enums := TypeSet.add x !enums ; - c_type_of_enum name ^ " *" + sprintf "enum %s *" (typename name) | Option n -> c_type_of_ty needed record n | x -> eprintf "%s" (Types.to_string x) ; assert false -and c_type_of_enum name = sprintf "enum %s " (typename name) - -and initialiser_of_ty = function - | SecretString | String | Ref _ | Set _ | Map _ | Record _ -> - " *result = NULL;\n" - | _ -> - "" - -and mapname l r = sprintf "%s_%s_map" (name_of_ty l) (name_of_ty r) - and name_of_ty = function | SecretString | String -> "string" @@ -1262,21 +884,7 @@ and name_of_ty = function eprintf "%s" (Types.to_string x) ; assert false -and decl_filename name = - let dir = - if Astring.String.is_suffix ~affix:"internal" name then "" else "xen/api/" - in - sprintf "%sxen_%s.h" dir (String.lowercase_ascii name) - -and predecl_filename name = - sprintf "xen/api/xen_%s_decl.h" (String.lowercase_ascii name) - -and internal_decl_filename name = - sprintf "xen_%s_internal.h" (String.lowercase_ascii name) - -and impl_filename name = sprintf "xen_%s.c" (String.lowercase_ascii name) - -and protector classname = sprintf "XEN_%s_H" (String.uppercase_ascii classname) +and mapname l r = sprintf "%s_%s_map" (name_of_ty l) (name_of_ty r) and typename classname = sprintf "xen_%s" (String.lowercase_ascii classname) @@ -1284,16 +892,6 @@ and record_typename classname = sprintf "%s_record" (typename classname) and record_opt_typename classname = sprintf "%s_record_opt" (typename classname) -and messagename classname name = - sprintf "xen_%s_%s" - (String.lowercase_ascii classname) - (String.lowercase_ascii name) - -and messagename_async classname name = - sprintf "xen_%s_%s_async" - (String.lowercase_ascii classname) - (String.lowercase_ascii name) - and keyword_map name = let keywords = [("class", "XEN_CLAZZ"); ("public", "pubblic")] in if List.mem_assoc name keywords then List.assoc name keywords else name @@ -1302,14 +900,6 @@ and paramname name = keyword_map (String.lowercase_ascii name) and fieldname name = keyword_map (String.lowercase_ascii name) -and print_h_header out_chan protect = - let print format = fprintf out_chan format in - print "%s\n\n" Licence.bsd_two_clause ; - print "#ifndef %s\n" protect ; - print "#define %s\n\n" protect - -and print_h_footer out_chan = fprintf out_chan "\n#endif\n" - and populate_version () = List.iter (fun x -> render_file x json_releases templates_dir destdir) @@ -1319,4 +909,4 @@ and populate_version () = ; ("xen_api_version.c.mustache", "src/xen_api_version.c") ] -let _ = main () ; populate_version () +let _ = main () ; gen_failure () ; populate_version () diff --git a/ocaml/sdk-gen/c/templates/Makefile.mustache b/ocaml/sdk-gen/c/templates/Makefile.mustache index 0718072223a..ac78e5ca1e6 100644 --- a/ocaml/sdk-gen/c/templates/Makefile.mustache +++ b/ocaml/sdk-gen/c/templates/Makefile.mustache @@ -97,3 +97,4 @@ clean: .PHONY: clean build install .DEFAULT_GOAL := build + diff --git a/ocaml/sdk-gen/c/templates/class.c.mustache b/ocaml/sdk-gen/c/templates/class.c.mustache new file mode 100644 index 00000000000..55f6da267ae --- /dev/null +++ b/ocaml/sdk-gen/c/templates/class.c.mustache @@ -0,0 +1,192 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include +#include + +{{#internal_headers}} +#include "xen_{{header}}.h" +{{/internal_headers}} +{{#headers}} +#include +{{/headers}} + + +{{^is_event}} +XEN_FREE(xen_{{{class_lower}}}) +XEN_SET_ALLOC_FREE(xen_{{{class_lower}}}) +{{/is_event}} +XEN_ALLOC(xen_{{{class_lower}}}_record) +XEN_SET_ALLOC_FREE(xen_{{{class_lower}}}_record) +{{^is_event}} +XEN_ALLOC(xen_{{{class_lower}}}_record_opt) +XEN_RECORD_OPT_FREE(xen_{{{class_lower}}}) +XEN_SET_ALLOC_FREE(xen_{{{class_lower}}}_record_opt) +{{/is_event}} + + +static const struct_member xen_{{{class_lower}}}_record_struct_members[] = + { +{{#fields}} + { .key = "{{{field_name}}}", + .type = &{{{abstract_field_type}}}, + .offset = offsetof(xen_{{{class_lower}}}_record, {{{field_name_lower}}}) }{{^is_last}},{{/is_last}} +{{/fields}} + }; + + +const abstract_type xen_{{{class_lower}}}_record_abstract_type_ = + { + .XEN_API_TYPE = STRUCT, + .struct_size = sizeof(xen_{{{class_lower}}}_record), + .member_count = + sizeof(xen_{{{class_lower}}}_record_struct_members) / sizeof(struct_member), + .members = xen_{{{class_lower}}}_record_struct_members + }; + + +const abstract_type xen_{{{class_lower}}}_record_set_abstract_type_ = + { + .XEN_API_TYPE = SET, + .child = &xen_{{{class_lower}}}_record_abstract_type_ + }; +{{#has_all_records}} + + +static const struct struct_member xen_{{{class_lower}}}_xen_{{{class_lower}}}_record_members[] = +{ + { + .type = &abstract_type_string, + .offset = offsetof(xen_{{{class_lower}}}_xen_{{{class_lower}}}_record_map_contents, key) + }, + { + .type = &xen_{{{class_lower}}}_record_abstract_type_, + .offset = offsetof(xen_{{{class_lower}}}_xen_{{{class_lower}}}_record_map_contents, val) + } +}; + + +const abstract_type abstract_type_string_xen_{{{class_lower}}}_record_map = +{ + .XEN_API_TYPE = MAP, + .struct_size = sizeof(xen_{{{class_lower}}}_xen_{{{class_lower}}}_record_map_contents), + .members = xen_{{{class_lower}}}_xen_{{{class_lower}}}_record_members +}; +{{/has_all_records}} + + +void +xen_{{{class_lower}}}_record_free(xen_{{{class_lower}}}_record *record) +{ + if (record == NULL) + return; + +{{^is_event}} + free(record->handle); +{{/is_event}} +{{#fields}} +{{#can_free}} + {{{free_record_field}}} +{{/can_free}} +{{/fields}} + free(record); +} +{{#messages}} + + +bool +xen_{{{class_lower}}}_{{{msg_name_lower}}}(xen_session *session{{#sync_params}}, {{{param_type}}}{{{param_name}}}{{/sync_params}}) +{ +{{#has_params}} + abstract_value param_values[] = + { +{{#msg_params}} + { .type = &{{{abstract_param_type}}}, + .u.{{{abstract_member}}}_val = {{{abstract_member_conv}}} }{{^is_last}},{{/is_last}} +{{/msg_params}} + }; +{{/has_params}} +{{#has_result}} + + abstract_type result_type = {{{abstract_result_type}}}; +{{/has_result}} + +{{#init_result}} + *result = NULL; +{{/init_result}} +{{#has_result}} +{{#has_params}} + XEN_CALL_("{{{class_name}}}.{{{msg_name}}}"); +{{/has_params}} +{{^has_params}} + xen_call_(session, "{{{class_name}}}.{{{msg_name}}}", NULL, 0, &result_type, result); +{{/has_params}} +{{/has_result}} +{{^has_result}} + xen_call_(session, "{{{class_name}}}.{{{msg_name}}}", {{#has_params}}param_values{{/has_params}}{{^has_params}}NULL{{/has_params}}, {{param_count}}, NULL, NULL); +{{/has_result}} +{{#is_result_record}} + + if (session->ok) + (*result)->handle = xen_strdup_((*result)->uuid); + +{{/is_result_record}} + return session->ok; +} +{{#is_async}} + + +bool +xen_{{{class_lower}}}_{{{msg_name_lower}}}_async(xen_session *session{{#async_params}}, {{{param_type}}}{{{param_name}}}{{/async_params}}) +{ +{{#has_params}} + abstract_value param_values[] = + { +{{#msg_params}} + { .type = &{{{abstract_param_type}}}, + .u.{{{abstract_member}}}_val = {{{abstract_member_conv}}} }{{^is_last}},{{/is_last}} +{{/msg_params}} + }; +{{/has_params}} + + abstract_type result_type = abstract_type_string; + + *result = NULL; +{{#has_params}} + XEN_CALL_("Async.{{{class_name}}}.{{{msg_name}}}"); +{{/has_params}} +{{^has_params}} + xen_call_(session, "Async.{{{class_name}}}.{{{msg_name}}}", NULL, 0, &result_type, result); +{{/has_params}} + return session->ok; +} +{{/is_async}} +{{/messages}} + diff --git a/ocaml/sdk-gen/c/templates/class.h.mustache b/ocaml/sdk-gen/c/templates/class.h.mustache new file mode 100644 index 00000000000..98dd1f37446 --- /dev/null +++ b/ocaml/sdk-gen/c/templates/class.h.mustache @@ -0,0 +1,179 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef XEN_{{{class_upper}}}_H +#define XEN_{{{class_upper}}}_H + +{{#headers}} +#include +{{/headers}} + + +{{{class_doc}}} + + +{{^is_event}} +/** + * Free the given xen_{{{class_lower}}}. The given handle must have been + * allocated by this library. + */ +extern void +xen_{{{class_lower}}}_free(xen_{{{class_lower}}} {{{class_lower}}}); + + +typedef struct xen_{{{class_lower}}}_set +{ + size_t size; + xen_{{{class_lower}}} *contents[]; +} xen_{{{class_lower}}}_set; + +/** + * Allocate a xen_{{{class_lower}}}_set of the given size. + */ +extern xen_{{{class_lower}}}_set * +xen_{{{class_lower}}}_set_alloc(size_t size); + +/** + * Free the given xen_{{{class_lower}}}_set. The given set must have been + * allocated by this library. + */ +extern void +xen_{{{class_lower}}}_set_free(xen_{{{class_lower}}}_set *set); + + +{{/is_event}} +typedef struct xen_{{{class_lower}}}_record +{ +{{^is_event}} + xen_{{{class_lower}}} handle; +{{/is_event}} +{{#fields}} + {{{field_type}}}{{{field_name_lower}}}; +{{/fields}} +} xen_{{{class_lower}}}_record; + +/** + * Allocate a xen_{{{class_lower}}}_record. + */ +extern xen_{{{class_lower}}}_record * +xen_{{{class_lower}}}_record_alloc(void); + +/** + * Free the given xen_{{{class_lower}}}_record, and all referenced values. + * The given record must have been allocated by this library. + */ +extern void +xen_{{{class_lower}}}_record_free(xen_{{{class_lower}}}_record *record); + + +{{^is_event}} +typedef struct xen_{{{class_lower}}}_record_opt +{ + bool is_record; + union + { + xen_{{{class_lower}}} handle; + xen_{{{class_lower}}}_record *record; + } u; +} xen_{{{class_lower}}}_record_opt; + +/** + * Allocate a xen_{{{class_lower}}}_record_opt. + */ +extern xen_{{{class_lower}}}_record_opt * +xen_{{{class_lower}}}_record_opt_alloc(void); + +/** + * Free the given xen_{{{class_lower}}}_record_opt, and all referenced values. + * The given record_opt must have been allocated by this library. + */ +extern void +xen_{{{class_lower}}}_record_opt_free(xen_{{{class_lower}}}_record_opt *record_opt); + + +{{/is_event}} +typedef struct xen_{{{class_lower}}}_record_set +{ + size_t size; + xen_{{{class_lower}}}_record *contents[]; +} xen_{{{class_lower}}}_record_set; + +/** + * Allocate a xen_{{{class_lower}}}_record_set of the given size. + */ +extern xen_{{{class_lower}}}_record_set * +xen_{{{class_lower}}}_record_set_alloc(size_t size); + +/** + * Free the given xen_{{{class_lower}}}_record_set, and all referenced values. + * The given set must have been allocated by this library. + */ +extern void +xen_{{{class_lower}}}_record_set_free(xen_{{{class_lower}}}_record_set *set); + + +{{^is_event}} +typedef struct xen_{{{class_lower}}}_record_opt_set +{ + size_t size; + xen_{{{class_lower}}}_record_opt *contents[]; +} xen_{{{class_lower}}}_record_opt_set; + +/** + * Allocate a xen_{{{class_lower}}}_record_opt_set of the given size. + */ +extern xen_{{{class_lower}}}_record_opt_set * +xen_{{{class_lower}}}_record_opt_set_alloc(size_t size); + +/** + * Free the given xen_{{{class_lower}}}_record_opt_set, and all referenced + * values. The given set must have been allocated by this library. + */ +extern void +xen_{{{class_lower}}}_record_opt_set_free(xen_{{{class_lower}}}_record_opt_set *set); + + +{{/is_event}} +{{#messages}} +{{{msg_doc}}} +extern bool +xen_{{{class_lower}}}_{{{msg_name_lower}}}(xen_session *session{{#sync_params}}, {{{param_type}}}{{{param_name}}}{{/sync_params}}); + + +{{#is_async}} +{{{msg_doc}}} +extern bool +xen_{{{class_lower}}}_{{{msg_name_lower}}}_async(xen_session *session{{#async_params}}, {{{param_type}}}{{{param_name}}}{{/async_params}}); + + +{{/is_async}} +{{/messages}} +#endif + diff --git a/ocaml/sdk-gen/c/templates/class_decl.h.mustache b/ocaml/sdk-gen/c/templates/class_decl.h.mustache new file mode 100644 index 00000000000..521d3d49d40 --- /dev/null +++ b/ocaml/sdk-gen/c/templates/class_decl.h.mustache @@ -0,0 +1,47 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef XEN_{{{class_upper}}}_DECL_H +#define XEN_{{{class_upper}}}_DECL_H + +{{^is_event}} +typedef void *xen_{{{class_lower}}}; + +struct xen_{{{class_lower}}}_set; +{{/is_event}} +struct xen_{{{class_lower}}}_record; +struct xen_{{{class_lower}}}_record_set; +{{^is_event}} +struct xen_{{{class_lower}}}_record_opt; +struct xen_{{{class_lower}}}_record_opt_set; +{{/is_event}} + +#endif + diff --git a/ocaml/sdk-gen/c/templates/map.c.mustache b/ocaml/sdk-gen/c/templates/map.c.mustache new file mode 100644 index 00000000000..0b944b35ad3 --- /dev/null +++ b/ocaml/sdk-gen/c/templates/map.c.mustache @@ -0,0 +1,92 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +{{#internal_headers}} +#include "xen_{{{header}}}.h" +{{/internal_headers}} +{{#headers}} +#include +{{/headers}} + + +xen_{{{map_lower}}} * +xen_{{{map_lower}}}_alloc(size_t size) +{ + xen_{{{map_lower}}} *result = + calloc(1, sizeof(xen_{{{map_lower}}}) + + size * sizeof(struct xen_{{{map_lower}}}_contents)); + result->size = size; + return result; +} + + +void +xen_{{{map_lower}}}_free(xen_{{{map_lower}}} *map) +{ +{{#can_free}} + if (map == NULL) + return; + + size_t n = map->size; + for (size_t i = 0; i < n; i++) + { +{{#can_free_key}} + {{{free_key}}} +{{/can_free_key}} +{{#can_free_val}} + {{{free_val}}} +{{/can_free_val}} + } + +{{/can_free}} + free(map); +} +{{#enum_map}} + + +static const struct_member {{{map_lower}}}_struct_members[] = + { + { .type = &{{{abstract_type_key}}}, + .offset = offsetof(xen_{{{map_lower}}}_contents, key) }, + { .type = &{{{abstract_type_val}}}, + .offset = offsetof(xen_{{{map_lower}}}_contents, val) }, + }; + + +const abstract_type {{{map_lower}}}_abstract_type_ = + { + .XEN_API_TYPE = MAP, + .struct_size = sizeof({{{map_lower}}}_struct_members), + .member_count = + sizeof({{{map_lower}}}_struct_members) / sizeof(struct_member), + .members = {{{map_lower}}}_struct_members + }; +{{/enum_map}} + diff --git a/ocaml/sdk-gen/c/templates/map.h.mustache b/ocaml/sdk-gen/c/templates/map.h.mustache new file mode 100644 index 00000000000..aa7c96bf512 --- /dev/null +++ b/ocaml/sdk-gen/c/templates/map.h.mustache @@ -0,0 +1,68 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef XEN_{{{map_upper}}}_H +#define XEN_{{{map_upper}}}_H + + +{{#headers}} +#include +{{/headers}} + + +typedef struct xen_{{{map_lower}}}_contents +{ + {{{key_type_lower}}}key; + {{{val_type_lower}}}val; +} xen_{{{map_lower}}}_contents; + + +typedef struct xen_{{{map_lower}}} +{ + size_t size; + xen_{{{map_lower}}}_contents contents[]; +} xen_{{{map_lower}}}; + +/** + * Allocate a xen_{{{map_lower}}} of the given size. + */ +extern xen_{{{map_lower}}} * +xen_{{{map_lower}}}_alloc(size_t size); + +/** + * Free the given xen_{{{map_lower}}} + * and all referenced values. The map must have been allocated by this library. + */ +extern void +xen_{{{map_lower}}}_free(xen_{{{map_lower}}} *map); + + +#endif + diff --git a/ocaml/sdk-gen/c/templates/xen_all.h.mustache b/ocaml/sdk-gen/c/templates/xen_all.h.mustache index 9d9bef9143e..fb86a54f4ef 100644 --- a/ocaml/sdk-gen/c/templates/xen_all.h.mustache +++ b/ocaml/sdk-gen/c/templates/xen_all.h.mustache @@ -27,7 +27,6 @@ * OF THE POSSIBILITY OF SUCH DAMAGE. */ -/* This file is autogenerated */ #ifndef XEN_API_XEN_ALL_H #define XEN_API_XEN_ALL_H @@ -37,9 +36,10 @@ #include #include {{#api_headers}} -#include <{{api_header}}> +#include {{/api_headers}} #include #include #endif + diff --git a/ocaml/sdk-gen/c/templates/xen_api_failure.c.mustache b/ocaml/sdk-gen/c/templates/xen_api_failure.c.mustache new file mode 100644 index 00000000000..f35926bfce1 --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_api_failure.c.mustache @@ -0,0 +1,58 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#include "xen_internal.h" +#include + + +/* + * Maintain this in the same order as the enum declaration! + */ +static const char *lookup_table[] = +{ +{{#api_errors}} + "{{api_error}}", +{{/api_errors}} +}; + + +const char * +xen_api_failure_to_string(enum xen_api_failure val) +{ + return lookup_table[val]; +} + + +extern enum xen_api_failure +xen_api_failure_from_string(const char *str) +{ + return ENUM_LOOKUP(str, lookup_table); +} + diff --git a/ocaml/sdk-gen/c/templates/xen_api_failure.h.mustache b/ocaml/sdk-gen/c/templates/xen_api_failure.h.mustache new file mode 100644 index 00000000000..3094d7a51ea --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_api_failure.h.mustache @@ -0,0 +1,66 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef XEN_API_FAILURE_H +#define XEN_API_FAILURE_H + + +enum xen_api_failure +{ +{{#api_errors}} +{{{api_error_doc}}} + XEN_API_FAILURE_{{api_error}}, + +{{/api_errors}} + /** + * Unknown to this SDK version. + */ + XEN_API_FAILURE_UNDEFINED +}; + + +/** + * Return the name corresponding to the given code. This string must + * not be modified or freed. + */ +extern const char * +xen_api_failure_to_string(enum xen_api_failure val); + + +/** + * Return the correct code for the given string, or UNDEFINED if the + * given string does not match a known code. + */ +extern enum xen_api_failure +xen_api_failure_from_string(const char *str); + + +#endif + diff --git a/ocaml/sdk-gen/c/templates/xen_api_version.c.mustache b/ocaml/sdk-gen/c/templates/xen_api_version.c.mustache index 0a13575d334..94b0c894b47 100644 --- a/ocaml/sdk-gen/c/templates/xen_api_version.c.mustache +++ b/ocaml/sdk-gen/c/templates/xen_api_version.c.mustache @@ -27,6 +27,7 @@ * OF THE POSSIBILITY OF SUCH DAMAGE. */ + #include "xen/api/xen_api_version.h" const char * @@ -53,3 +54,4 @@ xen_api_version_from_int(int64_t major_version, int64_t minor_version) {{/releases}} return xen_api_unknown_version; } + diff --git a/ocaml/sdk-gen/c/templates/xen_api_version.h.mustache b/ocaml/sdk-gen/c/templates/xen_api_version.h.mustache index 5f55ec79291..09115486aa6 100644 --- a/ocaml/sdk-gen/c/templates/xen_api_version.h.mustache +++ b/ocaml/sdk-gen/c/templates/xen_api_version.h.mustache @@ -27,6 +27,7 @@ * OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef XEN_API_VERSION_H #define XEN_API_VERSION_H @@ -48,3 +49,4 @@ extern xen_api_version xen_api_version_from_int(int64_t major_version, int64_t minor_version); #endif + diff --git a/ocaml/sdk-gen/c/templates/xen_enum.c.mustache b/ocaml/sdk-gen/c/templates/xen_enum.c.mustache index 421c9015a6f..90b1d200868 100644 --- a/ocaml/sdk-gen/c/templates/xen_enum.c.mustache +++ b/ocaml/sdk-gen/c/templates/xen_enum.c.mustache @@ -96,3 +96,4 @@ const abstract_type xen_{{{enum_name}}}_set_abstract_type_ = {{/event_operations}} + diff --git a/ocaml/sdk-gen/c/templates/xen_enum.h.mustache b/ocaml/sdk-gen/c/templates/xen_enum.h.mustache index 824179cf2d3..3a944a71438 100644 --- a/ocaml/sdk-gen/c/templates/xen_enum.h.mustache +++ b/ocaml/sdk-gen/c/templates/xen_enum.h.mustache @@ -38,14 +38,12 @@ enum xen_{{{enum_name}}} { {{#enum_values}} - /** - * {{{enum_value_doc}}} - */ +{{{enum_value_doc}}} XEN_{{{enum_name_upper}}}_{{{enum_value_upper}}}, {{/enum_values}} /** - * Unknown to this version of the bindings. + * Unknown to this SDK version. */ XEN_{{{enum_name_upper}}}_UNDEFINED }; @@ -64,8 +62,8 @@ extern xen_{{{enum_name}}}_set * xen_{{{enum_name}}}_set_alloc(size_t size); /** - * Free the given xen_{{{enum_name}}}_set. The given set must have been - * allocated by this library. + * Free the given xen_{{{enum_name}}}_set. The given set must + * have been allocated by this library. */ extern void xen_{{{enum_name}}}_set_free(xen_{{{enum_name}}}_set *set); @@ -89,3 +87,4 @@ xen_{{{enum_name}}}_from_string(xen_session *session, const char *str); #endif + diff --git a/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache b/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache index b9731686edb..f3945be9738 100644 --- a/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache +++ b/ocaml/sdk-gen/c/templates/xen_enum_internal.h.mustache @@ -28,10 +28,9 @@ */ - /* * Declarations of the abstract types used during demarshalling of enum - * xen_{{{enum_name}}}. Internal to this library -- do not use from outside. + * xen_{{{enum_name}}}. */ @@ -43,9 +42,8 @@ extern const abstract_type xen_{{{enum_name}}}_abstract_type_; -{{^event_operations}} extern const abstract_type xen_{{{enum_name}}}_set_abstract_type_; -{{/event_operations}} #endif + diff --git a/ocaml/sdk-gen/c/templates/xen_enum_map_internal.h.mustache b/ocaml/sdk-gen/c/templates/xen_enum_map_internal.h.mustache new file mode 100644 index 00000000000..6d595ad16fc --- /dev/null +++ b/ocaml/sdk-gen/c/templates/xen_enum_map_internal.h.mustache @@ -0,0 +1,39 @@ +/* + * Copyright (c) Cloud Software Group, Inc. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1) Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2) Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + */ + + +#ifndef XEN_{{{map_upper}}}_INTERNAL_H +#define XEN_{{{map_upper}}}_INTERNAL_H + + +extern const abstract_type {{{map_lower}}}_abstract_type_; + + +#endif + diff --git a/ocaml/sdk-gen/c/templates/xen_internal.mustache b/ocaml/sdk-gen/c/templates/xen_internal.mustache index 934ae5047e0..621617511ce 100644 --- a/ocaml/sdk-gen/c/templates/xen_internal.mustache +++ b/ocaml/sdk-gen/c/templates/xen_internal.mustache @@ -27,6 +27,7 @@ * OF THE POSSIBILITY OF SUCH DAMAGE. */ + #ifndef XEN_INTERNAL_H #define XEN_INTERNAL_H @@ -208,3 +209,4 @@ void type__ ## _record_opt_free(type__ ## _record_opt *opt) { \ #endif + From e6afe15bfde080bd204c36ccb9c2773002166f98 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Thu, 11 Apr 2024 00:50:21 +0100 Subject: [PATCH 081/149] Removed erroneously ported recipe. Signed-off-by: Konstantina Chremmou --- Makefile | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index d7f3db993be..43ebc394d85 100644 --- a/Makefile +++ b/Makefile @@ -117,15 +117,10 @@ sdk: sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/csharp sh ocaml/sdk-gen/windows-line-endings.sh $(XAPISDK)/powershell -.PHONY: sdk-build-c sdk sdksanity +.PHONY: sdk-build-c sdk-build-c: sdk - cd _build/install/default/xapi/sdk/c && make -j $(JOBS) - -# workaround for no .resx generation, just for compilation testing -sdksanity: sdk - sed -i 's/FriendlyErrorNames.ResourceManager/null/g' ./_build/install/default/xapi/sdk/csharp/src/Failure.cs - cd _build/install/default/xapi/sdk/csharp/src && dotnet add package Newtonsoft.Json && dotnet build -f netstandard2.0 + cd _build/install/default/xapi/sdk/c && make clean && make -j $(JOBS) .PHONY: sdk-build-java From 92276c5df956e691c48a6b84442fc5e30c9edc83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Fri, 15 Dec 2023 17:05:04 +0000 Subject: [PATCH 082/149] CP-47033: Protocol_{lwt,async}: process requests concurrently (optional) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Protocol_unix spawns a new thread for each request and already processes requests concurrently. However Protocol_{lwt,async} was waiting for each request to be fully processed before fetching and dispatching the next one. Change `iter` to `iter_donwait`, run all promises in parallel, and don't wait for any of them to complete. Exceptions have to go somewhere though, so set up an Lwt.async_exception_hook that logs the exception. Note: this might need some throttling so that sending lots of RPC calls doesn't cause the server to run OOM.But Protocol_unix already has this problem. Signed-off-by: Edwin Török --- ocaml/message-switch/async/protocol_async.ml | 3 +++ ocaml/message-switch/core/s.ml | 2 ++ ocaml/message-switch/lwt/protocol_lwt.ml | 2 ++ ocaml/message-switch/switch/switch_main.ml | 7 +++++++ 4 files changed, 14 insertions(+) diff --git a/ocaml/message-switch/async/protocol_async.ml b/ocaml/message-switch/async/protocol_async.ml index 9ec9cc42b74..8c81a1e66d0 100644 --- a/ocaml/message-switch/async/protocol_async.ml +++ b/ocaml/message-switch/async/protocol_async.ml @@ -30,6 +30,9 @@ module M = struct let iter f t = Deferred.List.iter t ~f + let iter_dontwait f t = + Deferred.don't_wait_for @@ Deferred.List.iter ~how:`Parallel t ~f + let any = Deferred.any let is_determined = Deferred.is_determined diff --git a/ocaml/message-switch/core/s.ml b/ocaml/message-switch/core/s.ml index f99e0582687..d1f98e99aee 100644 --- a/ocaml/message-switch/core/s.ml +++ b/ocaml/message-switch/core/s.ml @@ -29,6 +29,8 @@ module type BACKEND = sig val iter : ('a -> unit t) -> 'a list -> unit t + val iter_dontwait : ('a -> unit t) -> 'a list -> unit + val any : 'a t list -> 'a t val is_determined : 'a t -> bool diff --git a/ocaml/message-switch/lwt/protocol_lwt.ml b/ocaml/message-switch/lwt/protocol_lwt.ml index 6da59eb3212..2ba53ed69cd 100644 --- a/ocaml/message-switch/lwt/protocol_lwt.ml +++ b/ocaml/message-switch/lwt/protocol_lwt.ml @@ -27,6 +27,8 @@ module M = struct let iter = Lwt_list.iter_s + let iter_dontwait f lst = Lwt.async (fun () -> Lwt_list.iter_p f lst) + let any = Lwt.choose let is_determined t = Lwt.state t <> Lwt.Sleep diff --git a/ocaml/message-switch/switch/switch_main.ml b/ocaml/message-switch/switch/switch_main.ml index 9bf78973a85..583baf6e594 100644 --- a/ocaml/message-switch/switch/switch_main.ml +++ b/ocaml/message-switch/switch/switch_main.ml @@ -75,6 +75,13 @@ module Lwt_result = struct let ( >>= ) m f = m >>= fun x -> f (Stdlib.Result.get_ok x) end +let exn_hook e = + let bt = Printexc.get_raw_backtrace () in + error "Caught exception in Lwt.async: %s" (Printexc.to_string e) ; + error "backtrace: %s" (Printexc.raw_backtrace_to_string bt) + +let () = Lwt.async_exception_hook := exn_hook + let make_server config trace_config = let open Config in info "Started server on %s" config.path ; From 8b81958c393d5dc473786fb8e78374d20dae8559 Mon Sep 17 00:00:00 2001 From: Vincent Liu Date: Wed, 20 Dec 2023 13:22:50 +0000 Subject: [PATCH 083/149] CP-47033: Make message switch concurrent processing optional Previously we had problems with making message switch process multiple messages at the same time, especially with the xapi-storage-script. It would still be good to have this feature, but be more conservative and make it off by default. Signed-off-by: Vincent Liu --- ocaml/message-switch/core/dune | 1 + ocaml/message-switch/core/make.ml | 92 ++++++++++++++++++++++ ocaml/message-switch/core/s.ml | 8 ++ ocaml/message-switch/unix/protocol_unix.ml | 2 + 4 files changed, 103 insertions(+) diff --git a/ocaml/message-switch/core/dune b/ocaml/message-switch/core/dune index 676fa3f20ee..4874cf7e439 100644 --- a/ocaml/message-switch/core/dune +++ b/ocaml/message-switch/core/dune @@ -9,6 +9,7 @@ sexplib sexplib0 uri + xapi-log ) (preprocess (pps ppx_deriving_rpc ppx_sexp_conv)) ) diff --git a/ocaml/message-switch/core/make.ml b/ocaml/message-switch/core/make.ml index 54e8904e1a9..224012909ac 100644 --- a/ocaml/message-switch/core/make.ml +++ b/ocaml/message-switch/core/make.ml @@ -16,6 +16,10 @@ open Sexplib.Std open Protocol +module D = Debug.Make (struct let name = "Message_switch.make" end) + +open D + module Connection = functor (IO : Cohttp.S.IO) @@ -406,4 +410,92 @@ functor in let _ = loop c None in return (Ok t) + + let listen_p ~process ~switch:port ~queue:name () = + let token = Printf.sprintf "%d" (Unix.getpid ()) in + let protect_connect path f = + M.connect path >>= fun conn -> + f conn >>= function + | Ok _ as ok -> + return ok + | Error _ as err -> + M.disconnect conn >>= fun () -> return err + in + let reconnect () = + protect_connect port @@ fun request_conn -> + Connection.rpc request_conn (In.Login token) >>|= fun (_ : string) -> + protect_connect port @@ fun reply_conn -> + Connection.rpc reply_conn (In.Login token) >>|= fun (_ : string) -> + return (Ok (request_conn, reply_conn)) + in + reconnect () >>|= fun ((request_conn, reply_conn) as c) -> + let request_shutdown = M.Ivar.create () in + let on_shutdown = M.Ivar.create () in + let mutex = M.Mutex.create () in + Connection.rpc request_conn (In.CreatePersistent name) >>|= fun _ -> + let t = {request_shutdown; on_shutdown} in + let reconnect () = + M.disconnect request_conn >>= fun () -> + M.disconnect reply_conn >>= reconnect + in + let rec loop c from = + let transfer = {In.from; timeout; queues= [name]} in + let frame = In.Transfer transfer in + let message = Connection.rpc request_conn frame in + any [map (fun _ -> ()) message; M.Ivar.read request_shutdown] + >>= fun () -> + if is_determined (M.Ivar.read request_shutdown) then ( + M.Ivar.fill on_shutdown () ; return (Ok ()) + ) else + message >>= function + | Error _e -> + M.Mutex.with_lock mutex reconnect >>|= fun c -> loop c from + | Ok raw -> ( + let transfer = Out.transfer_of_rpc (Jsonrpc.of_string raw) in + let print_error = function + | Ok (_ : string) -> + return () + | Error _ as err -> + error "message switch reply received error" ; + ignore @@ error_to_msg err ; + return () + in + match transfer.Out.messages with + | [] -> + loop c from + | _ :: _ -> + iter_dontwait + (fun (i, m) -> + process m.Message.payload >>= fun response -> + ( match m.Message.kind with + | Message.Response _ -> + return () (* configuration error *) + | Message.Request reply_to -> + let request = + In.Send + ( reply_to + , { + Message.kind= Message.Response i + ; payload= response + } + ) + in + M.Mutex.with_lock mutex (fun () -> + Connection.rpc reply_conn request + ) + >>= print_error + ) + >>= fun () -> + let request = In.Ack i in + M.Mutex.with_lock mutex (fun () -> + Connection.rpc reply_conn request + ) + >>= print_error + ) + transfer.Out.messages ; + loop c (Some transfer.Out.next) + ) + in + let _ = loop c None in + return (Ok t) end diff --git a/ocaml/message-switch/core/s.ml b/ocaml/message-switch/core/s.ml index d1f98e99aee..726e34aea92 100644 --- a/ocaml/message-switch/core/s.ml +++ b/ocaml/message-switch/core/s.ml @@ -91,6 +91,14 @@ module type SERVER = sig (** Connect to [switch] and start processing messages on [queue] via function [process] *) + val listen_p : + process:(string -> string io) + -> switch:string + -> queue:string + -> unit + -> t result io + (** same as above, but processes requests concurrently *) + val shutdown : t:t -> unit -> unit io (** [shutdown t] shutdown a server *) end diff --git a/ocaml/message-switch/unix/protocol_unix.ml b/ocaml/message-switch/unix/protocol_unix.ml index 678b302ab5a..485964a40ec 100644 --- a/ocaml/message-switch/unix/protocol_unix.ml +++ b/ocaml/message-switch/unix/protocol_unix.ml @@ -546,5 +546,7 @@ module Server = struct let (_ : Thread.t) = thread_forever (loop connections) None in Ok () + let listen_p = listen + let shutdown ~t:_ () = failwith "Shutdown is unimplemented" end From 966b6368e39cfee07d727746184f5be0e9a1541d Mon Sep 17 00:00:00 2001 From: Vincent Liu Date: Mon, 29 Jan 2024 17:06:05 +0000 Subject: [PATCH 084/149] CP-47033: Add test for concurrent message switch server Signed-off-by: Vincent Liu --- .../core_test/async/server_async_main.ml | 12 ++++- .../core_test/basic-rpc-test.sh | 6 ++- .../core_test/concur-rpc-test.sh | 45 +++++++++++++++++++ ocaml/message-switch/core_test/dune | 17 +++++++ .../core_test/lwt/server_main.ml | 16 ++++++- 5 files changed, 91 insertions(+), 5 deletions(-) create mode 100755 ocaml/message-switch/core_test/concur-rpc-test.sh diff --git a/ocaml/message-switch/core_test/async/server_async_main.ml b/ocaml/message-switch/core_test/async/server_async_main.ml index 2372cb34c98..cd7984bec27 100644 --- a/ocaml/message-switch/core_test/async/server_async_main.ml +++ b/ocaml/message-switch/core_test/async/server_async_main.ml @@ -23,6 +23,8 @@ let path = ref "/var/run/message-switch/sock" let name = ref "server" +let concurrent = ref false + let shutdown = Ivar.create () let process = function @@ -33,7 +35,10 @@ let process = function let main () = let (_ : 'a Deferred.t) = - Server.listen ~process ~switch:!path ~queue:!name () + if !concurrent then + Server.listen_p ~process ~switch:!path ~queue:!name () + else + Server.listen ~process ~switch:!path ~queue:!name () in Ivar.read shutdown >>= fun () -> Clock.after (Time.Span.of_sec 1.) >>= fun () -> exit 0 @@ -49,6 +54,11 @@ let _ = , Arg.Set_string name , Printf.sprintf "name to send message to (default %s)" !name ) + ; ( "-concurrent" + , Arg.Set concurrent + , Printf.sprintf "set concurrent processing of messages (default %b)" + !concurrent + ) ] (fun x -> P.fprintf stderr "Ignoring unexpected argument: %s" x) "Respond to RPCs on a name" ; diff --git a/ocaml/message-switch/core_test/basic-rpc-test.sh b/ocaml/message-switch/core_test/basic-rpc-test.sh index e73c3a873d1..877790370a2 100755 --- a/ocaml/message-switch/core_test/basic-rpc-test.sh +++ b/ocaml/message-switch/core_test/basic-rpc-test.sh @@ -1,12 +1,14 @@ #!/bin/bash set -e -SPATH=${TMPDIR:-/tmp}/sock -SWITCHPATH=${TMPDIR:-/tmp}/switch +SPATH=${TMPDIR:-/tmp}/sock_s +SWITCHPATH=${TMPDIR:-/tmp}/switch_s rm -rf ${SWITCHPATH} && mkdir -p ${SWITCHPATH} +echo Test message switch serial processing + echo Checking the switch can start late ./server_unix_main.exe -path $SPATH & sleep 1 diff --git a/ocaml/message-switch/core_test/concur-rpc-test.sh b/ocaml/message-switch/core_test/concur-rpc-test.sh new file mode 100755 index 00000000000..a91768972fe --- /dev/null +++ b/ocaml/message-switch/core_test/concur-rpc-test.sh @@ -0,0 +1,45 @@ +#!/bin/bash +set -e + +SPATH="${TMPDIR:-/tmp}/sock_p-$$" +SWITCHPATH="${TMPDIR:-/tmp}/switch_p-$$" + +trap "cleanup" TERM INT + +function cleanup { + rm -rf "${SWITCHPATH}" +} + +rm -rf "${SWITCHPATH}" && mkdir -p "${SWITCHPATH}" + +echo Test message switch concurrent processing + +echo Checking the switch can start late +test -x ./server_unix_main.exe || exit 1 +./server_unix_main.exe -path "$SPATH" & +sleep 1 +test -x ../switch/switch_main.exe && test -x ./client_unix_main.exe || exit 1 +../switch/switch_main.exe --path "$SPATH" --statedir "${SWITCHPATH}" & +./client_unix_main.exe -path "$SPATH" -secs 5 +sleep 2 + +echo Performance test of Lwt to Lwt +test -x lwt/server_main.exe && test -x lwt/client_main.exe || exit 1 +lwt/server_main.exe -path "$SPATH" -concurrent & +lwt/client_main.exe -path "$SPATH" -secs 5 +sleep 2 + +echo Performance test of Async to Lwt +test -x lwt/server_main.exe && test -x async/client_async_main.exe || exit 1 +lwt/server_main.exe -path "$SPATH" -concurrent & +async/client_async_main.exe -path "$SPATH" -secs 5 +sleep 2 + +echo Performance test of Async to Async +test -x async/server_async_main.exe && test -x async/client_async_main.exe || exit 1 +async/server_async_main.exe -path "$SPATH" -concurrent & +async/client_async_main.exe -path "$SPATH" -secs 5 +sleep 2 + +../cli/main.exe shutdown --path "$SPATH" +sleep 2 diff --git a/ocaml/message-switch/core_test/dune b/ocaml/message-switch/core_test/dune index d500c101354..dd7d00472f8 100644 --- a/ocaml/message-switch/core_test/dune +++ b/ocaml/message-switch/core_test/dune @@ -27,3 +27,20 @@ (package message-switch) ) +(rule + (alias runtest) + (deps + client_unix_main.exe + server_unix_main.exe + async/client_async_main.exe + async/server_async_main.exe + lwt/client_main.exe + lwt/server_main.exe + lwt/link_test_main.exe + ../switch/switch_main.exe + ../cli/main.exe + ) + (action (run ./concur-rpc-test.sh)) + (package message-switch) +) + diff --git a/ocaml/message-switch/core_test/lwt/server_main.ml b/ocaml/message-switch/core_test/lwt/server_main.ml index c30021ff35d..ece423dcb74 100644 --- a/ocaml/message-switch/core_test/lwt/server_main.ml +++ b/ocaml/message-switch/core_test/lwt/server_main.ml @@ -20,6 +20,8 @@ let path = ref "/var/run/message-switch/sock" let name = ref "server" +let concurrent = ref false + let t, u = Lwt.task () let process = function @@ -29,8 +31,13 @@ let process = function return x let main () = - Message_switch_lwt.Protocol_lwt.Server.listen ~process ~switch:!path - ~queue:!name () + ( if !concurrent then + Message_switch_lwt.Protocol_lwt.Server.listen_p ~process ~switch:!path + ~queue:!name () + else + Message_switch_lwt.Protocol_lwt.Server.listen ~process ~switch:!path + ~queue:!name () + ) >>= fun _ -> t >>= fun () -> Lwt_unix.sleep 1. @@ -45,6 +52,11 @@ let _ = , Arg.Set_string name , Printf.sprintf "name to send message to (default %s)" !name ) + ; ( "-concurrent" + , Arg.Set concurrent + , Printf.sprintf "set concurrent processing of messages (default %b)" + !concurrent + ) ] (fun x -> Printf.fprintf stderr "Ignoring unexpected argument: %s" x) "Respond to RPCs on a name" ; From 790e6fe346a4062f5150f70d07d027db990cdc18 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 11 Apr 2024 13:20:00 +0100 Subject: [PATCH 085/149] Remove mention of `dotnet-packages` in `sdk-gen`'s README Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ocaml/sdk-gen/README.md b/ocaml/sdk-gen/README.md index 7473d141f83..fb4d71650bf 100644 --- a/ocaml/sdk-gen/README.md +++ b/ocaml/sdk-gen/README.md @@ -12,9 +12,7 @@ The Python module is not auto-generated, it can be found at [XenAPI.py](../../scripts/examples/python/XenAPI/XenAPI.py). To compile the generated source code, follow the instructions in the corresponding -README files. The (patched) third party libraries required for the compilation -of the C# and PowerShell source code can be obtained from -[xenserver/dotnet-packages](https://github.com/xenserver/dotnet-packages) +`README` files. The repository [xenserver/xenserver-samples](https://github.com/xenserver/xenserver-samples) contains a number of examples for each of the five programming languages to help From e3d22994b2c51cd72738481e5534ac2cb4c88965 Mon Sep 17 00:00:00 2001 From: Danilo Del Busso Date: Thu, 11 Apr 2024 13:20:28 +0100 Subject: [PATCH 086/149] CP-48768: Update Folder Structure section in PS SDK's READMEs Signed-off-by: Danilo Del Busso --- ocaml/sdk-gen/powershell/autogen/README.md | 4 ++-- ocaml/sdk-gen/powershell/autogen/README_51.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ocaml/sdk-gen/powershell/autogen/README.md b/ocaml/sdk-gen/powershell/autogen/README.md index cbe06791bad..abbb3b0b1e7 100644 --- a/ocaml/sdk-gen/powershell/autogen/README.md +++ b/ocaml/sdk-gen/powershell/autogen/README.md @@ -51,9 +51,9 @@ The XenServer PowerShell Module is dependent upon the following libraries: This archive contains the following folders that are relevant to PowerShell users: -- `XenServerPowerShell\XenServerPSModule`: this is the XenServer PowerShell +- `XenServerPowerShell\PowerShell_7\XenServerPSModule`: this is the XenServer PowerShell Module -- `XenServerPowerShell\src`: contains the C# source code for the XenServer +- `XenServerPowerShell\PowerShell_7\src`: contains the C# source code for the XenServer cmdlets shipped as a Visual Studio project. ## Getting Started diff --git a/ocaml/sdk-gen/powershell/autogen/README_51.md b/ocaml/sdk-gen/powershell/autogen/README_51.md index 8088982ff47..4d5b19e26be 100644 --- a/ocaml/sdk-gen/powershell/autogen/README_51.md +++ b/ocaml/sdk-gen/powershell/autogen/README_51.md @@ -51,9 +51,9 @@ The XenServer PowerShell Module is dependent upon the following libraries: This archive contains the following folders that are relevant to PowerShell users: -- `XenServerPowerShell\XenServerPSModule`: this is the XenServer PowerShell +- `XenServerPowerShell\PowerShell_51\XenServerPSModule`: this is the XenServer PowerShell Module -- `XenServerPowerShell\src`: contains the C# source code for the XenServer +- `XenServerPowerShell\PowerShell_51\src`: contains the C# source code for the XenServer cmdlets shipped as a Visual Studio project. ## Getting Started From cee08ade7870875300af30faae54a1800edc14e5 Mon Sep 17 00:00:00 2001 From: Steven Woods Date: Thu, 11 Apr 2024 16:18:41 +0100 Subject: [PATCH 087/149] CA-391485: Avoid InterpolationSyntaxError by turning off interpolation OTEL_RESOURCE_ATTRIBUTES is in W3CBaggageFormat and thus uses %. ConfigParser attempts to interpolate these % signs but we want them to be left alone, so set interpolation=None Signed-off-by: Steven Woods --- python3/packages/observer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python3/packages/observer.py b/python3/packages/observer.py index 1eee0f2a7d6..b257aa9b3f0 100644 --- a/python3/packages/observer.py +++ b/python3/packages/observer.py @@ -71,7 +71,7 @@ def _get_configs_list(config_dir): def read_config(config_path, header): """Read a config file and return a dictionary of key-value pairs.""" - parser = configparser.ConfigParser() + parser = configparser.ConfigParser(interpolation=None) with open(config_path, encoding="utf-8") as config_file: try: parser.read_string(f"[{header}]\n{config_file.read()}") From f4aadeb688c16fe0e1f1a416c09577b77c53c5a8 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Fri, 12 Apr 2024 11:32:57 +0100 Subject: [PATCH 088/149] opam: add xapi-log to message-switch-core dependencies A recent change added the dependency Signed-off-by: Pau Ruiz Safont --- message-switch-core.opam | 1 + message-switch-core.opam.template | 1 + 2 files changed, 2 insertions(+) diff --git a/message-switch-core.opam b/message-switch-core.opam index 960934bea54..44e2983cc5b 100644 --- a/message-switch-core.opam +++ b/message-switch-core.opam @@ -22,6 +22,7 @@ depends: [ "ppx_sexp_conv" "rpclib" "sexplib" + "xapi-log" ] synopsis: "A simple store-and-forward message switch" description: """ diff --git a/message-switch-core.opam.template b/message-switch-core.opam.template index 7ec11e91dc3..7f65fa07598 100644 --- a/message-switch-core.opam.template +++ b/message-switch-core.opam.template @@ -20,6 +20,7 @@ depends: [ "ppx_sexp_conv" "rpclib" "sexplib" + "xapi-log" ] synopsis: "A simple store-and-forward message switch" description: """ From e505d89be33ac959c4631d168af3f8123960b7a2 Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Fri, 12 Apr 2024 11:18:58 +0000 Subject: [PATCH 089/149] Remove _t suffix for syslog_stdout_t type There's no need for suffix, Ocaml has different namespace for types. This was suggested as part of a code review. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/lib/fe.ml | 4 ++-- ocaml/forkexecd/lib/forkhelpers.ml | 2 +- ocaml/forkexecd/lib/forkhelpers.mli | 8 ++++---- ocaml/forkexecd/src/child.ml | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ocaml/forkexecd/lib/fe.ml b/ocaml/forkexecd/lib/fe.ml index 1a176a62baa..c928cd3fc10 100644 --- a/ocaml/forkexecd/lib/fe.ml +++ b/ocaml/forkexecd/lib/fe.ml @@ -1,13 +1,13 @@ (* Disable "Warning 39: unused rec flag." caused by rpc *) [@@@warning "-39"] -type syslog_stdout_t = {enabled: bool; key: string option} [@@deriving rpc] +type syslog_stdout = {enabled: bool; key: string option} [@@deriving rpc] type setup_cmd = { cmdargs: string list ; env: string list ; id_to_fd_map: (string * int option) list - ; syslog_stdout: syslog_stdout_t + ; syslog_stdout: syslog_stdout ; redirect_stderr_to_stdout: bool } [@@deriving rpc] diff --git a/ocaml/forkexecd/lib/forkhelpers.ml b/ocaml/forkexecd/lib/forkhelpers.ml index 15fb4bca6c5..f212ae7f03d 100644 --- a/ocaml/forkexecd/lib/forkhelpers.ml +++ b/ocaml/forkexecd/lib/forkhelpers.ml @@ -171,7 +171,7 @@ let with_logfile_fd ?(delete = true) prefix f = exception Spawn_internal_error of string * string * Unix.process_status -type syslog_stdout_t = +type syslog_stdout = | NoSyslogging | Syslog_DefaultKey | Syslog_WithKey of string diff --git a/ocaml/forkexecd/lib/forkhelpers.mli b/ocaml/forkexecd/lib/forkhelpers.mli index 48832726c02..186cbe51872 100644 --- a/ocaml/forkexecd/lib/forkhelpers.mli +++ b/ocaml/forkexecd/lib/forkhelpers.mli @@ -34,7 +34,7 @@ (** {2 High-level interface } *) -type syslog_stdout_t = +type syslog_stdout = | NoSyslogging | Syslog_DefaultKey | Syslog_WithKey of string @@ -45,7 +45,7 @@ val default_path_env_pair : string array val execute_command_get_output : ?env:string array - -> ?syslog_stdout:syslog_stdout_t + -> ?syslog_stdout:syslog_stdout -> ?redirect_stderr_to_stdout:bool -> ?timeout:float -> string @@ -57,7 +57,7 @@ val execute_command_get_output : val execute_command_get_output_send_stdin : ?env:string array - -> ?syslog_stdout:syslog_stdout_t + -> ?syslog_stdout:syslog_stdout -> ?redirect_stderr_to_stdout:bool -> ?timeout:float -> string @@ -97,7 +97,7 @@ val safe_close_and_exec : -> Unix.file_descr option -> Unix.file_descr option -> (string * Unix.file_descr) list - -> ?syslog_stdout:syslog_stdout_t + -> ?syslog_stdout:syslog_stdout -> ?redirect_stderr_to_stdout:bool -> string -> string list diff --git a/ocaml/forkexecd/src/child.ml b/ocaml/forkexecd/src/child.ml index 197f3b91f65..0bdb5fc1dc1 100644 --- a/ocaml/forkexecd/src/child.ml +++ b/ocaml/forkexecd/src/child.ml @@ -3,13 +3,13 @@ let debug (fmt : ('a, unit, string, unit) format4) = exception Cancelled -type syslog_stdout_t = {enabled: bool; key: string option} +type syslog_stdout = {enabled: bool; key: string option} type state_t = { cmdargs: string list ; env: string list ; id_to_fd_map: (string * int option) list - ; syslog_stdout: syslog_stdout_t + ; syslog_stdout: syslog_stdout ; redirect_stderr_to_stdout: bool ; ids_received: (string * Unix.file_descr) list ; fd_sock2: Unix.file_descr option From 97e8c14af9a9dde45fc4e25d7661b6949ea103ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edwin=20T=C3=B6r=C3=B6k?= Date: Tue, 12 Mar 2024 17:04:51 +0000 Subject: [PATCH 090/149] CA-389929: xenopsd: fix Xen version comparison. 4.17 is > 4.2, not lower! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't use string comparisons for versions, "17" < "2", but 17 > 2! Signed-off-by: Edwin Török --- ocaml/xenopsd/lib/xenops_utils.ml | 4 ++-- ocaml/xenopsd/xc/xenops_server_xen.ml | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/ocaml/xenopsd/lib/xenops_utils.ml b/ocaml/xenopsd/lib/xenops_utils.ml index 9a6ae66a9f7..d948f9865d9 100644 --- a/ocaml/xenopsd/lib/xenops_utils.ml +++ b/ocaml/xenopsd/lib/xenops_utils.ml @@ -584,7 +584,7 @@ let _sys_hypervisor_version_major = "/sys/hypervisor/version/major" let _sys_hypervisor_version_minor = "/sys/hypervisor/version/minor" type hypervisor = - | Xen of string * string + | Xen of int * int (* major, minor *) | Other of string @@ -601,7 +601,7 @@ let detect_hypervisor () = let minor = String.trim (Unixext.string_of_file _sys_hypervisor_version_minor) in - Some (Xen (major, minor)) + Some (Xen (int_of_string major, int_of_string minor)) | x -> Some (Other x) else diff --git a/ocaml/xenopsd/xc/xenops_server_xen.ml b/ocaml/xenopsd/xc/xenops_server_xen.ml index 4de4bbc3573..2f60e3b2716 100644 --- a/ocaml/xenopsd/xc/xenops_server_xen.ml +++ b/ocaml/xenopsd/xc/xenops_server_xen.ml @@ -5147,11 +5147,10 @@ let init () = look_for_forkexec () ; let major, minor = look_for_xen () in look_for_xenctrl () ; - if - major < "4" || ((major = "4" && minor < "2") && !Xenopsd.run_hotplug_scripts) + if major < 4 || ((major = 4 && minor < 2) && !Xenopsd.run_hotplug_scripts) then ( error - "This is xen version %s.%s. On all versions < 4.1 we must use \ + "This is xen version %d.%d. On all versions < 4.2 we must use \ hotplug/udev scripts" major minor ; error From 0d1d99244e980d012e4b474709103dcc230d95ce Mon Sep 17 00:00:00 2001 From: Vincent Liu Date: Tue, 16 Jan 2024 14:24:14 +0000 Subject: [PATCH 091/149] Add test for lock implementation in message_switch The lock implementation for `protocol_async` was not correct, hence adding the test suggested by Edwin to trigger race. Test works by trying to acquire a standard Ocaml mutex in a critical section provided by the `with_lock` implementation. This means that any failed attempt to acquire the lock implies the `with_lock` implementation does not provide mutual exclusion. Signed-off-by: Vincent Liu --- ocaml/message-switch/async/protocol_async.ml | 19 +++++++++ ocaml/message-switch/async/protocol_async.mli | 2 + ocaml/message-switch/core/dune | 1 + ocaml/message-switch/core/mtest.ml | 42 +++++++++++++++++++ ocaml/message-switch/core/s.ml | 16 +++++++ ocaml/message-switch/core_test/dune | 30 +++++++++++++ .../core_test/lock_test_async.ml | 13 ++++++ .../message-switch/core_test/lock_test_lwt.ml | 5 +++ ocaml/message-switch/lwt/protocol_lwt.ml | 17 ++++++++ ocaml/message-switch/lwt/protocol_lwt.mli | 2 + quality-gate.sh | 6 +-- 11 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 ocaml/message-switch/core/mtest.ml create mode 100644 ocaml/message-switch/core_test/lock_test_async.ml create mode 100644 ocaml/message-switch/core_test/lock_test_lwt.ml diff --git a/ocaml/message-switch/async/protocol_async.ml b/ocaml/message-switch/async/protocol_async.ml index 8c81a1e66d0..5898d22f77f 100644 --- a/ocaml/message-switch/async/protocol_async.ml +++ b/ocaml/message-switch/async/protocol_async.ml @@ -35,7 +35,11 @@ module M = struct let any = Deferred.any + let all = Deferred.all + let is_determined = Deferred.is_determined + + let return_unit = Deferred.unit end let connect path = @@ -98,6 +102,20 @@ module M = struct ) end + module Condition = struct + open Async_kernel + + type 'a t = 'a Condition.t + + let create = Condition.create + + let wait = Condition.wait + + let broadcast = Condition.broadcast + + let signal = Condition.signal + end + module Clock = struct type timer = {cancel: unit Ivar.t} @@ -120,3 +138,4 @@ end module Client = Message_switch_core.Make.Client (M) module Server = Message_switch_core.Make.Server (M) +module Mtest = Message_switch_core.Mtest.Make (M) diff --git a/ocaml/message-switch/async/protocol_async.mli b/ocaml/message-switch/async/protocol_async.mli index f691c24c989..d18b37b742c 100644 --- a/ocaml/message-switch/async/protocol_async.mli +++ b/ocaml/message-switch/async/protocol_async.mli @@ -19,3 +19,5 @@ open Message_switch_core module Client : S.CLIENT with type 'a io = 'a Deferred.t module Server : S.SERVER with type 'a io = 'a Deferred.t + +module Mtest : Mtest.MTEST with type 'a io = 'a Deferred.t diff --git a/ocaml/message-switch/core/dune b/ocaml/message-switch/core/dune index 4874cf7e439..6debbc895c7 100644 --- a/ocaml/message-switch/core/dune +++ b/ocaml/message-switch/core/dune @@ -10,6 +10,7 @@ sexplib0 uri xapi-log + xapi-stdext-threads ) (preprocess (pps ppx_deriving_rpc ppx_sexp_conv)) ) diff --git a/ocaml/message-switch/core/mtest.ml b/ocaml/message-switch/core/mtest.ml new file mode 100644 index 00000000000..3b8da9803fe --- /dev/null +++ b/ocaml/message-switch/core/mtest.ml @@ -0,0 +1,42 @@ +module type MTEST = sig + type +'a io + + val mutex_provides_mutal_exclusion : unit -> unit io +end + +module Make = +functor + (M : S.BACKEND) + -> + struct + open M.IO + + type 'a io = 'a M.IO.t + + let ocaml_lock = Mutex.create () + + let mu = M.Mutex.create () + + let cond = M.Condition.create () + + let broadcast () = M.Condition.broadcast cond () + + let mutex_provides_mutal_exclusion () : unit io = + let promises = + List.init 100 (fun _ -> + M.Condition.wait cond >>= fun () -> + M.Mutex.with_lock mu (fun () -> + M.IO.return_unit >>= fun () -> + (* the with_lock implementation should ensure that only one + monad can try to acquire this lock *) + assert (Mutex.try_lock ocaml_lock) ; + M.IO.return_unit >>= fun () -> + Mutex.unlock ocaml_lock ; M.IO.return_unit + ) + ) + in + broadcast () ; + ignore @@ all promises ; + Printf.printf "%s test.\n" (M.whoami ()) ; + M.IO.return_unit + end diff --git a/ocaml/message-switch/core/s.ml b/ocaml/message-switch/core/s.ml index 726e34aea92..423304d1b24 100644 --- a/ocaml/message-switch/core/s.ml +++ b/ocaml/message-switch/core/s.ml @@ -33,7 +33,11 @@ module type BACKEND = sig val any : 'a t list -> 'a t + val all : 'a t list -> 'a list t + val is_determined : 'a t -> bool + + val return_unit : unit t end val connect : string -> (IO.ic * IO.oc) IO.t @@ -58,6 +62,18 @@ module type BACKEND = sig val with_lock : t -> (unit -> 'a IO.t) -> 'a IO.t end + module Condition : sig + type 'a t + + val create : unit -> 'a t + + val wait : 'a t -> 'a IO.t + + val broadcast : 'a t -> 'a -> unit + + val signal : 'a t -> 'a -> unit + end + module Clock : sig type timer diff --git a/ocaml/message-switch/core_test/dune b/ocaml/message-switch/core_test/dune index dd7d00472f8..449f2fae5c5 100644 --- a/ocaml/message-switch/core_test/dune +++ b/ocaml/message-switch/core_test/dune @@ -3,13 +3,43 @@ (names client_unix_main server_unix_main + lock_test_async + lock_test_lwt + ) + (modules + client_unix_main + server_unix_main + lock_test_async + lock_test_lwt ) (libraries message-switch-unix + message-switch-core + message-switch-async + message-switch-lwt threads.posix ) ) +(rule + (alias runtest) + (deps + lock_test_async.exe + ) + (action (run ./lock_test_async.exe)) + (package message-switch) +) + +(rule + (alias runtest) + (deps + lock_test_lwt.exe + ) + (action (run ./lock_test_lwt.exe)) + (package message-switch) +) + + (rule (alias runtest) (deps diff --git a/ocaml/message-switch/core_test/lock_test_async.ml b/ocaml/message-switch/core_test/lock_test_async.ml new file mode 100644 index 00000000000..85cde8eaecb --- /dev/null +++ b/ocaml/message-switch/core_test/lock_test_async.ml @@ -0,0 +1,13 @@ +open Core +open Async +open Message_switch_async + +let ( >>= ) = Deferred.( >>= ) + +let test_async_lock () = Protocol_async.Mtest.mutex_provides_mutal_exclusion () + +let () = + don't_wait_for + (test_async_lock () >>= fun () -> shutdown 0 ; Deferred.return ()) + +let () = never_returns (Scheduler.go ()) diff --git a/ocaml/message-switch/core_test/lock_test_lwt.ml b/ocaml/message-switch/core_test/lock_test_lwt.ml new file mode 100644 index 00000000000..784599dafa4 --- /dev/null +++ b/ocaml/message-switch/core_test/lock_test_lwt.ml @@ -0,0 +1,5 @@ +open Message_switch_lwt + +let test_lwt_lock = Protocol_lwt.Mtest.mutex_provides_mutal_exclusion () + +let () = Lwt_main.run test_lwt_lock diff --git a/ocaml/message-switch/lwt/protocol_lwt.ml b/ocaml/message-switch/lwt/protocol_lwt.ml index 2ba53ed69cd..26c9c874d55 100644 --- a/ocaml/message-switch/lwt/protocol_lwt.ml +++ b/ocaml/message-switch/lwt/protocol_lwt.ml @@ -31,7 +31,11 @@ module M = struct let any = Lwt.choose + let all = Lwt.all + let is_determined t = Lwt.state t <> Lwt.Sleep + + let return_unit = Lwt.return_unit end let connect path = @@ -77,6 +81,18 @@ module M = struct let with_lock = Lwt_mutex.with_lock end + module Condition = struct + type 'a t = 'a Lwt_condition.t + + let create = Lwt_condition.create + + let signal = Lwt_condition.signal + + let wait c = Lwt_condition.wait c + + let broadcast = Lwt_condition.broadcast + end + module Clock = struct type timer = unit Lwt.t @@ -92,3 +108,4 @@ end module Client = Message_switch_core.Make.Client (M) module Server = Message_switch_core.Make.Server (M) +module Mtest = Message_switch_core.Mtest.Make (M) diff --git a/ocaml/message-switch/lwt/protocol_lwt.mli b/ocaml/message-switch/lwt/protocol_lwt.mli index c9bd220155d..64ca15c0e8e 100644 --- a/ocaml/message-switch/lwt/protocol_lwt.mli +++ b/ocaml/message-switch/lwt/protocol_lwt.mli @@ -19,3 +19,5 @@ open Message_switch_core module Client : S.CLIENT with type 'a io = 'a Lwt.t module Server : S.SERVER with type 'a io = 'a Lwt.t + +module Mtest : Mtest.MTEST with type 'a io = 'a Lwt.t diff --git a/quality-gate.sh b/quality-gate.sh index 56e53e75b56..01fdd3e4094 100755 --- a/quality-gate.sh +++ b/quality-gate.sh @@ -25,10 +25,10 @@ verify-cert () { } mli-files () { - N=522 + N=516 # do not count ml files from the tests in ocaml/{tests/perftest/quicktest} - MLIS=$(git ls-files -- '**/*.mli' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) - MLS=$(git ls-files -- '**/*.ml' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) + MLIS=$(git ls-files -- '**/*.mli' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest|ocaml/message-switch/core_test" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) + MLS=$(git ls-files -- '**/*.ml' | grep -vE "ocaml/tests|ocaml/perftest|ocaml/quicktest|ocaml/message-switch/core_test" | xargs -I {} sh -c "echo {} | cut -f 1 -d '.'" \;) num_mls_without_mlis=$(comm -23 <(sort <<<"$MLS") <(sort <<<"$MLIS") | wc -l) if [ "$num_mls_without_mlis" -eq "$N" ]; then echo "OK counted $num_mls_without_mlis .ml files without an .mli" From d26f0b82e12d5e2c7af64a542e5dd56b8e3f9efd Mon Sep 17 00:00:00 2001 From: Frediano Ziglio Date: Sun, 7 Apr 2024 10:12:53 +0000 Subject: [PATCH 092/149] Check elapsed time for timeout test Not only print the time taken for program execution but also check the process was stopped not too early and not too late. Signed-off-by: Frediano Ziglio --- ocaml/forkexecd/test/fe_test.ml | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/ocaml/forkexecd/test/fe_test.ml b/ocaml/forkexecd/test/fe_test.ml index 1df62eadc28..42991d5f16b 100644 --- a/ocaml/forkexecd/test/fe_test.ml +++ b/ocaml/forkexecd/test/fe_test.ml @@ -109,14 +109,23 @@ let test_delay () = let start = Unix.gettimeofday () in let exe = Printf.sprintf "/proc/%d/exe" (Unix.getpid ()) in let args = ["sleep"] in + (* Need to have fractional part because some internal usage split integer + and fractional and do computation. + Better to have a high fractional part (> 0.5) to more probably exceed + the unit. + *) + let timeout = 1.7 in try - Forkhelpers.execute_command_get_output ~timeout:4.0 exe args |> ignore ; + Forkhelpers.execute_command_get_output ~timeout exe args |> ignore ; failwith "Failed to timeout" with | Forkhelpers.Subprocess_timeout -> - Printf.printf "Caught timeout exception after %f seconds\n%!" - (Unix.gettimeofday () -. start) ; - () + let elapsed = Unix.gettimeofday () -. start in + Printf.printf "Caught timeout exception after %f seconds\n%!" elapsed ; + if elapsed < timeout then + failwith "Process exited too soon" ; + if elapsed > timeout +. 0.2 then + failwith "Excessive time elapsed" | e -> failwith (Printf.sprintf "Failed with unexpected exception: %s" @@ -261,7 +270,7 @@ let slave = function pid (List.length filtered) ls ) -let sleep () = Unix.sleep 5 ; Printf.printf "Ok\n" +let sleep () = Unix.sleep 3 ; Printf.printf "Ok\n" let echo out err = if out <> "" then print_endline out ; From eb5d0feaa1fe7774334f6e4c3e5f3495f7a0c821 Mon Sep 17 00:00:00 2001 From: Mark Syms Date: Wed, 27 Mar 2024 16:02:42 +0000 Subject: [PATCH 093/149] CP-47991: add CBT fields to the volume struct Signed-off-by: Mark Syms --- ocaml/xapi-storage/generator/lib/control.ml | 11 +++++++++++ ocaml/xapi-storage/generator/test/storage_test.ml | 2 ++ ocaml/xapi-storage/rpc-light/SR.ls/response | 2 ++ ocaml/xapi-storage/rpc-light/Volume.clone/response | 2 ++ ocaml/xapi-storage/rpc-light/Volume.create/response | 2 ++ ocaml/xapi-storage/rpc-light/Volume.snapshot/response | 2 ++ 6 files changed, 21 insertions(+) diff --git a/ocaml/xapi-storage/generator/lib/control.ml b/ocaml/xapi-storage/generator/lib/control.ml index 93b2800a766..f4d8a22a4a5 100644 --- a/ocaml/xapi-storage/generator/lib/control.ml +++ b/ocaml/xapi-storage/generator/lib/control.ml @@ -30,6 +30,12 @@ type health = (** Storage is busy recovering, e.g. rebuilding mirrors *) [@@deriving rpcty] +type volume_type = + | Data (** Normal data volume *) + | CBT_Metadata (** CBT Metadata only, data destroyed *) + | Data_and_CBT_Metadata (** Both Data and CBT Metadata *) +[@@deriving rpcty] + (** Primary key for a specific Storage Repository. This can be any string which is meaningful to the implementation. For example this could be an NFS directory name, an LVM VG name or even a URI. This string is @@ -116,6 +122,11 @@ type volume = { ; keys: (string * string) list (** A list of key=value pairs which have been stored in the Volume metadata. These should not be interpreted by the Volume plugin. *) + ; volume_type: volume_type option [@default Some Data] + (** The content type of this volume *) + ; cbt_enabled: bool option [@default Some false] + (** True means that the storage datapath will track changed dirty blocks + while writing and will be able to provide CBT Metadata when requested *) } [@@deriving rpcty] diff --git a/ocaml/xapi-storage/generator/test/storage_test.ml b/ocaml/xapi-storage/generator/test/storage_test.ml index eca6cf45afb..3da8be64711 100644 --- a/ocaml/xapi-storage/generator/test/storage_test.ml +++ b/ocaml/xapi-storage/generator/test/storage_test.ml @@ -57,6 +57,8 @@ let test_volume = ; physical_utilisation= 0L ; uri= ["uri1"] ; keys= [] + ; cbt_enabled= Some false + ; volume_type= Some Data } (** Check that we successfully parse the responses and diff --git a/ocaml/xapi-storage/rpc-light/SR.ls/response b/ocaml/xapi-storage/rpc-light/SR.ls/response index b85cff59c56..7f989e33066 100644 --- a/ocaml/xapi-storage/rpc-light/SR.ls/response +++ b/ocaml/xapi-storage/rpc-light/SR.ls/response @@ -12,6 +12,8 @@ physical_utilisation0 uriuri1 keys + volume_typeData + cbt_enabledfalse diff --git a/ocaml/xapi-storage/rpc-light/Volume.clone/response b/ocaml/xapi-storage/rpc-light/Volume.clone/response index 4b0f52b2305..dc4036f599d 100644 --- a/ocaml/xapi-storage/rpc-light/Volume.clone/response +++ b/ocaml/xapi-storage/rpc-light/Volume.clone/response @@ -11,6 +11,8 @@ physical_utilisation0 uriuri1 keys + volume_typeData + cbt_enabledfalse diff --git a/ocaml/xapi-storage/rpc-light/Volume.create/response b/ocaml/xapi-storage/rpc-light/Volume.create/response index 4b0f52b2305..dc4036f599d 100644 --- a/ocaml/xapi-storage/rpc-light/Volume.create/response +++ b/ocaml/xapi-storage/rpc-light/Volume.create/response @@ -11,6 +11,8 @@ physical_utilisation0 uriuri1 keys + volume_typeData + cbt_enabledfalse diff --git a/ocaml/xapi-storage/rpc-light/Volume.snapshot/response b/ocaml/xapi-storage/rpc-light/Volume.snapshot/response index 4b0f52b2305..dc4036f599d 100644 --- a/ocaml/xapi-storage/rpc-light/Volume.snapshot/response +++ b/ocaml/xapi-storage/rpc-light/Volume.snapshot/response @@ -11,6 +11,8 @@ physical_utilisation0 uriuri1 keys + volume_typeData + cbt_enabledfalse From d384dbcc945147e6ed28c53b8cd75301b8580b2c Mon Sep 17 00:00:00 2001 From: Gabriel Buica Date: Tue, 26 Mar 2024 11:46:02 +0000 Subject: [PATCH 094/149] CP-46576: Add standard network attributes Add standard http attributes to from opentelemetry such as: - `network.local.address`; - `network.local.port`; - `network.peer.address`; - `network.peer.port`. This improves debuggability associated with client requests. Signed-off-by: Gabriel Buica --- ocaml/xapi/context.ml | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/ocaml/xapi/context.ml b/ocaml/xapi/context.ml index fc611de7fb5..dce55ca4d40 100644 --- a/ocaml/xapi/context.ml +++ b/ocaml/xapi/context.ml @@ -229,6 +229,34 @@ let parent_of_origin (origin : origin) span_name = let attribute_helper_fn f v = Option.fold ~none:[] ~some:f v +let addr_port_of_sock s = + match s with + | None -> + (None, None) + | Some (Unix.ADDR_UNIX "") -> + (None, None) + | Some (Unix.ADDR_UNIX socket_name) -> + (Some socket_name, None) + | Some (Unix.ADDR_INET (addr, port)) -> + (Some (Unix.string_of_inet_addr addr), Some (string_of_int port)) + +let with_try_get_addr f s = + (try Some (f s) with Unix.Unix_error (Unix.ENOTSOCK, _, _) -> None) + |> addr_port_of_sock + +let attr_of_fd s = + let peer_addr, peer_port = s |> with_try_get_addr Unix.getpeername in + let local_addr, local_port = s |> with_try_get_addr Unix.getsockname in + [ + attribute_helper_fn + (fun addr -> [("network.local.address", addr)]) + local_addr + ; attribute_helper_fn (fun port -> [("network.local.port", port)]) local_port + ; attribute_helper_fn (fun addr -> [("network.peer.address", addr)]) peer_addr + ; attribute_helper_fn (fun port -> [("network.peer.port", port)]) peer_port + ] + |> List.concat + let attr_of_req (req : Http.Request.t) = [ [ @@ -277,8 +305,8 @@ let make_attributes ?task_name ?task_id ?task_uuid ?session_id ?origin () = match origin with | Internal -> [("xs.xapi.task.origin", "internal")] - | Http (req, _) -> - attr_of_req req + | Http (req, s) -> + [attr_of_req req; attr_of_fd s] |> List.concat ) origin ] From 74f7437485e1060e1236bdfbdbf795c38687e98d Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 17 Apr 2024 13:04:15 +0100 Subject: [PATCH 095/149] ocaml/idl: generate enum{_to_string,__all} functions This is useful for unit-testing, and more. Printing helps with displaying the values in unit-tests, and the all helps to get exhaustive checks without having to manually sync the types Signed-off-by: Pau Ruiz Safont --- ocaml/idl/ocaml_backend/gen_api.ml | 18 ++++++++++++++++++ ocaml/idl/ocaml_backend/ocaml_utils.ml | 13 ++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ocaml/idl/ocaml_backend/gen_api.ml b/ocaml/idl/ocaml_backend/gen_api.ml index 01c49bdbe88..734e78d6bb6 100644 --- a/ocaml/idl/ocaml_backend/gen_api.ml +++ b/ocaml/idl/ocaml_backend/gen_api.ml @@ -72,6 +72,23 @@ let overrides = ) ] +(** Generate enum__all and enum_to_string bindings for all enums *) +let gen_enum_helpers tys = + let gen_string_and_all = function + | DT.Set (DT.Enum (_, elist) as e) -> + let nlist = List.map fst elist in + [ + Printf.sprintf "let %s__all = %s" (OU.alias_of_ty e) + (OU.ocaml_list_of_enum nlist) + ; (Printf.sprintf "let %s_to_string = %s") + (OU.alias_of_ty e) + (OU.ocaml_to_string_of_enum nlist) + ] + | _ -> + [] + in + List.concat_map gen_string_and_all tys + (** Generate a single type declaration for simple types (eg not containing references to record objects) *) let gen_non_record_type tys = let rec aux accu = function @@ -382,6 +399,7 @@ let gen_client_types highapi = ; gen_non_record_type all_types ; gen_record_type ~with_module:true highapi (toposort_types highapi all_types) + ; gen_enum_helpers all_types ; O.Signature.strings_of (Gen_client.gen_signature highapi) ] ) diff --git a/ocaml/idl/ocaml_backend/ocaml_utils.ml b/ocaml/idl/ocaml_backend/ocaml_utils.ml index e3ab8ac19dd..a01ae955586 100644 --- a/ocaml/idl/ocaml_backend/ocaml_utils.ml +++ b/ocaml/idl/ocaml_backend/ocaml_utils.ml @@ -58,9 +58,15 @@ let ocaml_of_record_field = function let ocaml_of_module_name x = String.capitalize_ascii x +let ocaml_map_enum_ sep f list = String.concat sep (List.map f list) + (** Convert an IDL enum into a polymorhic variant. *) let ocaml_of_enum list = - "[ " ^ String.concat " | " (List.map constructor_of list) ^ " ]" + Printf.sprintf "[%s]" (ocaml_map_enum_ " | " constructor_of list) + +(* Create a to_string function for a polymorphic variant. *) +let ocaml_list_of_enum list = + Printf.sprintf "[%s]" (ocaml_map_enum_ "; " constructor_of list) (** Convert an IDL type to a function name; we need to generate functions to marshal/unmarshal from XML for each unique IDL type *) @@ -90,6 +96,11 @@ let rec alias_of_ty = function | Option x -> sprintf "%s_option" (alias_of_ty x) +(** Create the body of a to_string function for an enum *) +let ocaml_to_string_of_enum list = + let single name = Printf.sprintf {|%s -> "%s"|} (constructor_of name) name in + Printf.sprintf "function %s" (ocaml_map_enum_ " | " single list) + (** Convert an IDL type into a string containing OCaml code representing the type. *) let rec ocaml_of_ty = function From c20b11e7d6e6ab4e094e78bf3ebf02dea1856845 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Tue, 16 Apr 2024 15:40:50 +0100 Subject: [PATCH 096/149] test: add tests for allowed VM operations Signed-off-by: Pau Ruiz Safont --- ocaml/tests/test_vm_check_operation_error.ml | 125 +++++++++++-------- 1 file changed, 73 insertions(+), 52 deletions(-) diff --git a/ocaml/tests/test_vm_check_operation_error.ml b/ocaml/tests/test_vm_check_operation_error.ml index a91fdcfa229..0423338e630 100644 --- a/ocaml/tests/test_vm_check_operation_error.ml +++ b/ocaml/tests/test_vm_check_operation_error.ml @@ -1,54 +1,6 @@ -let all_vm_operations = - [ - `assert_operation_valid - ; `awaiting_memory_live - ; `call_plugin - ; `changing_VCPUs - ; `changing_VCPUs_live - ; `changing_dynamic_range - ; `changing_memory_limits - ; `changing_memory_live - ; `changing_shadow_memory - ; `changing_shadow_memory_live - ; `changing_static_range - ; `changing_NVRAM - ; `checkpoint - ; `clean_reboot - ; `clean_shutdown - ; `clone - ; `copy - ; `create_template - ; `csvm - ; `data_source_op - ; `destroy - ; `export - ; `get_boot_record - ; `hard_reboot - ; `hard_shutdown - ; `import - ; `make_into_template - ; `metadata_export - ; `migrate_send - ; `pause - ; `pool_migrate - ; `power_state_reset - ; `provision - ; `query_services - ; `resume - ; `resume_on - ; `revert - ; `reverting - ; `send_sysrq - ; `send_trigger - ; `shutdown - ; `snapshot - ; `snapshot_with_quiesce - ; `start - ; `start_on - ; `suspend - ; `unpause - ; `update_allowed_operations - ] +let vm_op_to_string = API.vm_operations_to_string + +let pp_vm_op () = Fmt.(str "%a" (of_to_string vm_op_to_string)) let with_test_vm f = let __context = Mock.make_context_with_new_db "Mock context" in @@ -75,7 +27,7 @@ let test_null_vdi () = ~strict:true ) ) - all_vm_operations + API.vm_operations__all ) let test_vm_set_nvram_running () = @@ -155,6 +107,71 @@ let test_sxm_allowed_when_rum () = ) ) +let test_is_allowed_concurrently (expected, (op, current_ops)) = + let ops_to_str ops = + String.concat "," (List.map (fun (_, op) -> vm_op_to_string op) ops) + in + let name = + match current_ops with + | [] -> + vm_op_to_string op + | lst -> + Printf.sprintf "%a when %s" pp_vm_op op (ops_to_str lst) + in + + let test () = + let actual = Xapi_vm_lifecycle.is_allowed_concurrently ~op ~current_ops in + let name = + Printf.sprintf "%a allowed in [%s]" pp_vm_op op (ops_to_str current_ops) + in + Alcotest.(check bool) name expected actual + in + (name, `Quick, test) + +let allowed_specs = + let current_of op = ((), op) in + let allow_hard_shutdown = + List.map + (fun op -> + let allowed = match op with `hard_shutdown -> false | _ -> true in + (allowed, (`hard_shutdown, [current_of op])) + ) + API.vm_operations__all + in + let allow_hard_reboot = + List.map + (fun op -> + let allowed = + match op with `hard_shutdown | `hard_reboot -> false | _ -> true + in + (allowed, (`hard_reboot, [current_of op])) + ) + API.vm_operations__all + in + let allow_clean_shutdown = + List.map + (fun op -> + let allowed = match op with `migrate_send -> true | _ -> false in + (allowed, (`clean_shutdown, [current_of op])) + ) + API.vm_operations__all + in + List.concat + [ + [ + (true, (`snapshot, [])) + ; (true, (`snapshot, [current_of `checkpoint])) + ; (false, (`migrate_send, [current_of `clean_reboot])) + ; (true, (`clean_reboot, [current_of `migrate_send])) + ] + ; allow_hard_shutdown + ; allow_clean_shutdown + ; allow_hard_reboot + ] + +let test_allow_concurrently = + List.map test_is_allowed_concurrently allowed_specs + let test = [ ("test_null_vdi", `Quick, test_null_vdi) @@ -166,3 +183,7 @@ let test = ; ("test_sxm_allowed_when_rum", `Quick, test_sxm_allowed_when_rum) ; ("test_vm_set_nvram when VM is running", `Quick, test_vm_set_nvram_running) ] + +let () = + Alcotest.run "Xapi_vm_lifecycle" + [("is_allowed_concurrently", test_allow_concurrently)] From 723a4983ab1555b24222b417350cddebbf6dc1f5 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Wed, 17 Apr 2024 13:50:54 +0100 Subject: [PATCH 097/149] ocaml/xapi: use generated enum list instead of hand-maintained ones The list in xapi_vm_lifecycle now contains ignored operations, this allows users to easily see which operations are excluded from the database, decide whether they're worth exposing them and change the code accordingly to accomodate for them. Signed-off-by: Pau Ruiz Safont --- ocaml/xapi/xapi_host_helpers.ml | 14 +------ ocaml/xapi/xapi_sr_operations.ml | 24 +----------- ocaml/xapi/xapi_vdi_helpers.ml | 21 +--------- ocaml/xapi/xapi_vm_lifecycle.ml | 67 ++++++++++++++------------------ 4 files changed, 32 insertions(+), 94 deletions(-) diff --git a/ocaml/xapi/xapi_host_helpers.ml b/ocaml/xapi/xapi_host_helpers.ml index 040f5782273..00f01d83ed2 100644 --- a/ocaml/xapi/xapi_host_helpers.ml +++ b/ocaml/xapi/xapi_host_helpers.ml @@ -26,19 +26,7 @@ let with_lock = Xapi_stdext_threads.Threadext.Mutex.execute let finally = Xapi_stdext_pervasives.Pervasiveext.finally -let all_operations = - [ - `provision - ; `evacuate - ; `reboot - ; `shutdown - ; `vm_start - ; `vm_resume - ; `vm_migrate - ; `power_on - ; `apply_updates - ; `enable - ] +let all_operations = API.host_allowed_operations__all (** Returns a table of operations -> API error options (None if the operation would be ok) *) let valid_operations ~__context record _ref' = diff --git a/ocaml/xapi/xapi_sr_operations.ml b/ocaml/xapi/xapi_sr_operations.ml index b44c8bf5916..8f7a7d8012a 100644 --- a/ocaml/xapi/xapi_sr_operations.ml +++ b/ocaml/xapi/xapi_sr_operations.ml @@ -26,29 +26,7 @@ open Client open Record_util -let all_ops : API.storage_operations_set = - [ - `scan - ; `destroy - ; `forget - ; `plug - ; `unplug - ; `vdi_create - ; `vdi_destroy - ; `vdi_resize - ; `vdi_clone - ; `vdi_snapshot - ; `vdi_mirror - ; `vdi_enable_cbt - ; `vdi_disable_cbt - ; `vdi_data_destroy - ; `vdi_list_changed_blocks - ; `vdi_set_on_boot - ; `vdi_introduce - ; `update - ; `pbd_create - ; `pbd_destroy - ] +let all_ops = API.storage_operations__all (* This list comes from https://github.com/xenserver/xen-api/blob/tampa-bugfix/ocaml/xapi/xapi_sr_operations.ml#L36-L38 *) let all_rpu_ops : API.storage_operations_set = diff --git a/ocaml/xapi/xapi_vdi_helpers.ml b/ocaml/xapi/xapi_vdi_helpers.ml index 2e3355ef1f4..6b4366a80ce 100644 --- a/ocaml/xapi/xapi_vdi_helpers.ml +++ b/ocaml/xapi/xapi_vdi_helpers.ml @@ -26,26 +26,7 @@ module D = Debug.Make (struct let name = "xapi_vdi_helpers" end) open D -let all_ops : API.vdi_operations_set = - [ - `blocked - ; `clone - ; `copy - ; `data_destroy - ; `destroy - ; `disable_cbt - ; `enable_cbt - ; `force_unlock - ; `forget - ; `generate_config - ; `list_changed_blocks - ; `mirror - ; `resize - ; `resize_online - ; `set_on_boot - ; `snapshot - ; `update - ] +let all_ops = API.vdi_operations__all (* CA-26514: Block operations on 'unmanaged' VDIs *) let assert_managed ~__context ~vdi = diff --git a/ocaml/xapi/xapi_vm_lifecycle.ml b/ocaml/xapi/xapi_vm_lifecycle.ml index 7d35a12f1d0..ccee66500cd 100644 --- a/ocaml/xapi/xapi_vm_lifecycle.ml +++ b/ocaml/xapi/xapi_vm_lifecycle.ml @@ -749,50 +749,41 @@ let vtpm_update_allowed_operations ~__context ~self = let allowed = match state with `Halted -> ops | _ -> [] in Db.VTPM.set_allowed_operations ~__context ~self ~value:allowed +let ignored_ops = + [ + `create_template + ; `power_state_reset + ; `csvm + ; `get_boot_record + ; `send_sysrq + ; `send_trigger + ; `query_services + ; `shutdown + ; `call_plugin + ; `changing_memory_live + ; `awaiting_memory_live + ; `changing_memory_limits + ; `changing_shadow_memory_live + ; `changing_VCPUs + ; `assert_operation_valid + ; `data_source_op + ; `update_allowed_operations + ; `import + ; `reverting + ] + +let allowable_ops = + List.filter (fun op -> not (List.mem op ignored_ops)) API.vm_operations__all + let update_allowed_operations ~__context ~self = - let check_operation_error = check_operation_error ~__context ~ref:self in let check accu op = - match check_operation_error ~op ~strict:true with + match check_operation_error ~__context ~ref:self ~op ~strict:true with | None -> op :: accu - | _ -> + | Some _err -> accu in - let allowed = - List.fold_left check [] - [ - `snapshot - ; `copy - ; `clone - ; `revert - ; `checkpoint - ; `snapshot_with_quiesce - ; `start - ; `start_on - ; `pause - ; `unpause - ; `clean_shutdown - ; `clean_reboot - ; `hard_shutdown - ; `hard_reboot - ; `suspend - ; `resume - ; `resume_on - ; `export - ; `destroy - ; `provision - ; `changing_VCPUs_live - ; `pool_migrate - ; `migrate_send - ; `make_into_template - ; `changing_static_range - ; `changing_shadow_memory - ; `changing_dynamic_range - ; `changing_NVRAM - ; `create_vtpm - ; `metadata_export - ] - in + let allowed = List.fold_left check [] allowable_ops in (* FIXME: need to be able to deal with rolling-upgrade for orlando as well *) let allowed = if Helpers.rolling_upgrade_in_progress ~__context then From 6681b7d265e79d1afd0d6ebca93edcca1aa6cc95 Mon Sep 17 00:00:00 2001 From: Konstantina Chremmou Date: Thu, 11 Apr 2024 01:15:57 +0100 Subject: [PATCH 098/149] Added github workflow to build and release the C SDK. Signed-off-by: Konstantina Chremmou --- .github/workflows/generate-and-build-sdks.yml | 30 +++++++++++++++++++ .github/workflows/release.yml | 20 +++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/.github/workflows/generate-and-build-sdks.yml b/.github/workflows/generate-and-build-sdks.yml index 80b32b5c8d9..db28438062f 100644 --- a/.github/workflows/generate-and-build-sdks.yml +++ b/.github/workflows/generate-and-build-sdks.yml @@ -24,6 +24,12 @@ jobs: shell: bash run: opam exec -- make sdk + - name: Store C SDK source + uses: actions/upload-artifact@v4 + with: + name: SDK_Source_C + path: _build/install/default/xapi/sdk/c/* + - name: Store C# SDK source uses: actions/upload-artifact@v4 with: @@ -39,6 +45,30 @@ jobs: - name: Cleanup XenAPI environment uses: ./.github/workflows/cleanup-xapi-environment + build-c-sdk: + name: Build C SDK + runs-on: ubuntu-latest + needs: generate-sdk-sources + steps: + - name: Install dependencies + run: sudo apt-get install libxml2-dev + + - name: Retrieve C SDK source + uses: actions/download-artifact@v4 + with: + name: SDK_Source_C + path: source/ + + - name: Build C SDK + shell: bash + run: make -C source + + - name: Store C SDK + uses: actions/upload-artifact@v4 + with: + name: SDK_Artifacts_C + path: source/* + build-csharp-sdk: name: Build C# SDK runs-on: windows-2022 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 87d5cc8721f..c4d133d2fa1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,6 +52,12 @@ jobs: name: XenAPI path: dist/ + - name: Retrieve C SDK distribution binaries + uses: actions/download-artifact@v4 + with: + name: SDK_Artifacts_C + path: libxenserver/usr/local/ + - name: Retrieve C# SDK distribution artifacts uses: actions/download-artifact@v4 with: @@ -70,10 +76,19 @@ jobs: name: SDK_Binaries_XenServerPowerShell_NET6 path: sdk_powershell_7x/ + - name: Package C SDK artifacts for deployment + shell: bash + run: | + mkdir -p libxenserver/usr/local/lib + mv libxenserver/usr/local/libxenserver.* libxenserver/usr/local/lib/ + tar -zcvf libxenserver-prerelease.tar.gz -C ./libxenserver usr/local/lib/ usr/local/include/xen/api + rm -rf libxenserver/usr/local/lib/ + tar -zcvf libxenserver-prerelease.src.tar.gz -C ./libxenserver/usr/local . + - name: Zip PowerShell 5.x SDK artifacts for deployment shell: bash run: zip PowerShell-SDK-5.x-prerelease-unsigned.zip ./sdk_powershell_5x -r - + - name: Zip PowerShell 7.x SDK artifacts for deployment shell: bash run: zip PowerShell-SDK-7.x-prerelease-unsigned.zip ./sdk_powershell_7x -r @@ -83,7 +98,8 @@ jobs: run: | gh release create ${{ github.ref_name }} --repo ${{ github.repository }} --generate-notes dist/* \ PowerShell-SDK-5.x-prerelease-unsigned.zip \ - PowerShell-SDK-7.x-prerelease-unsigned.zip + PowerShell-SDK-7.x-prerelease-unsigned.zip \ + libxenserver-prerelease.tar.gz libxenserver-prerelease.src.tar.gz env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From f753ee2dd624b63a1bb7903a17190892bf64ccb8 Mon Sep 17 00:00:00 2001 From: Pau Ruiz Safont Date: Thu, 18 Apr 2024 17:14:10 +0100 Subject: [PATCH 099/149] xenopsd: add mli to cli/xn and remove unused code Signed-off-by: Pau Ruiz Safont --- ocaml/xenopsd/cli/xn.ml | 297 --------------------------------------- ocaml/xenopsd/cli/xn.mli | 72 ++++++++++ quality-gate.sh | 2 +- 3 files changed, 73 insertions(+), 298 deletions(-) create mode 100644 ocaml/xenopsd/cli/xn.mli diff --git a/ocaml/xenopsd/cli/xn.ml b/ocaml/xenopsd/cli/xn.ml index ec883f3deed..0eb6ef5ac1b 100644 --- a/ocaml/xenopsd/cli/xn.ml +++ b/ocaml/xenopsd/cli/xn.ml @@ -65,74 +65,6 @@ let diagnose_error f = exit 1 ) -let usage () = - Printf.fprintf stderr - "%s [args] - send commands to the xenops daemon\n" Sys.argv.(0) ; - Printf.fprintf stderr "%s add - add a VM from \n" - Sys.argv.(0) ; - Printf.fprintf stderr "%s list [verbose] - query the states of known VMs\n" - Sys.argv.(0) ; - Printf.fprintf stderr "%s remove - forget about a VM\n" - Sys.argv.(0) ; - Printf.fprintf stderr "%s start [paused] - start a VM\n" - Sys.argv.(0) ; - Printf.fprintf stderr "%s pause - pause a VM\n" Sys.argv.(0) ; - Printf.fprintf stderr "%s unpause - unpause a VM\n" Sys.argv.(0) ; - Printf.fprintf stderr "%s shutdown - shutdown a VM\n" - Sys.argv.(0) ; - Printf.fprintf stderr "%s reboot - reboot a VM\n" Sys.argv.(0) ; - Printf.fprintf stderr "%s suspend - suspend a VM\n" - Sys.argv.(0) ; - Printf.fprintf stderr "%s resume - resume a VM\n" - Sys.argv.(0) ; - Printf.fprintf stderr - "%s migrate - migrate a VM to \n" Sys.argv.(0) ; - Printf.fprintf stderr - "%s vbd-list - query the states of a VM's block devices\n" - Sys.argv.(0) ; - Printf.fprintf stderr - "%s console-list - query the states of a VM's consoles\n" - Sys.argv.(0) ; - Printf.fprintf stderr - "%s pci-add - associate the PCI device \ - with \n" - Sys.argv.(0) ; - Printf.fprintf stderr - "%s pci-remove - disassociate the PCI device \ - with \n" - Sys.argv.(0) ; - Printf.fprintf stderr - "%s pci-list - query the states of a VM's PCI devices\n" - Sys.argv.(0) ; - Printf.fprintf stderr "%s cd-insert - insert a CD into a VBD\n" - Sys.argv.(0) ; - Printf.fprintf stderr "%s cd-eject - eject a CD from a VBD\n" - Sys.argv.(0) ; - Printf.fprintf stderr - "%s export-metadata - export metadata associated with \n" - Sys.argv.(0) ; - Printf.fprintf stderr - "%s export-metadata-xm - export metadata associated with in xm \ - format\n" - Sys.argv.(0) ; - Printf.fprintf stderr - "%s delay