Merge pull request #5851 from BengangY/private/bengangy/CP-49214

CP 49214: Upload and reposync from the bundle

Author: gangj

ocaml/idl/datamodel.ml

@@ -8432,6 +8432,7 @@ let http_actions =
       , []
       )
     )
+  ; ("put_bundle", (Put, Constants.put_bundle_uri, true, [], _R_POOL_OP, []))
   ]
 
 (* these public http actions will NOT be checked by RBAC *)

ocaml/idl/datamodel_errors.ml

@@ -1895,6 +1895,19 @@ let _ =
     ~doc:"The GPG public key file name in the repository is invalid." () ;
   error Api_errors.repository_already_exists ["ref"]
     ~doc:"The repository already exists." () ;
+  error Api_errors.bundle_repository_already_exists ["ref"]
+    ~doc:"The bundle repository already exists." () ;
+  error Api_errors.bundle_unpack_failed ["error"]
+    ~doc:"Failed to unpack bundle file." () ;
+  error Api_errors.bundle_repo_not_enabled []
+    ~doc:"Cannot sync bundle as the bundle repository is not enabled." () ;
+  error Api_errors.can_not_sync_updates []
+    ~doc:"Cannot sync updates as the bundle repository is enabled." () ;
+  error Api_errors.bundle_repo_should_be_single_enabled []
+    ~doc:
+      "If the bundle repository is enabled, it should be the only one enabled \
+       repository of the pool."
+    () ;
   error Api_errors.repository_is_in_use [] ~doc:"The repository is in use." () ;
   error Api_errors.repository_cleanup_failed []
     ~doc:"Failed to clean up local repository on coordinator." () ;
@@ -1907,6 +1920,11 @@ let _ =
       "The operation could not be performed because syncing updates is in \
        progress."
     () ;
+  error Api_errors.sync_bundle_in_progress []
+    ~doc:
+      "The operation could not be performed because syncing bundle is in \
+       progress."
+    () ;
   error Api_errors.reposync_failed []
     ~doc:"Syncing with remote YUM repository failed." () ;
   error Api_errors.invalid_repomd_xml [] ~doc:"The repomd.xml is invalid." () ;

ocaml/idl/datamodel_pool.ml

@@ -21,6 +21,9 @@ let operations =
       ; ( "sync_updates"
         , "Indicates this pool is in the process of syncing updates"
         )
+      ; ( "sync_bundle"
+        , "Indicates this pool is in the process of syncing bundle"
+        )
       ; ( "get_updates"
         , "Indicates this pool is in the process of getting updates"
         )

ocaml/idl/schematest.ml

@@ -3,7 +3,7 @@ let hash x = Digest.string x |> Digest.to_hex
 (* BEWARE: if this changes, check that schema has been bumped accordingly in
    ocaml/idl/datamodel_common.ml, usually schema_minor_vsn *)
 
-let last_known_schema_hash = "2a6baa01032827a321845b264c6aaae4"
+let last_known_schema_hash = "4417b0087b481c3038e73f170b7d4d01"
 
 let current_schema_hash : string =
   let open Datamodel_types in

ocaml/tests/dune

@@ -9,7 +9,7 @@
       test_vm_placement test_vm_helpers test_repository test_repository_helpers
       test_ref test_vm_group
       test_livepatch test_rpm test_updateinfo test_storage_smapiv1_wrapper test_storage_quicktest test_observer
-      test_pool_periodic_update_sync test_pkg_mgr test_tar_ext))
+      test_pool_periodic_update_sync test_pkg_mgr test_tar_ext test_pool_repository))
   (libraries
     alcotest
     angstrom
@@ -79,13 +79,13 @@
 (tests
   (names test_vm_helpers test_vm_placement test_network_sriov test_vdi_cbt
     test_clustering test_pusb test_daemon_manager test_repository test_repository_helpers
-    test_livepatch test_rpm test_updateinfo test_pool_periodic_update_sync test_pkg_mgr test_tar_ext)
+    test_livepatch test_rpm test_updateinfo test_pool_periodic_update_sync test_pkg_mgr test_tar_ext test_pool_repository)
   (package xapi)
   (modes exe)
   (modules test_vm_helpers test_vm_placement test_network_sriov test_vdi_cbt
     test_event test_clustering test_cluster_host test_cluster test_pusb
     test_daemon_manager test_repository test_repository_helpers test_livepatch test_rpm
-    test_updateinfo test_pool_periodic_update_sync test_pkg_mgr test_tar_ext)
+    test_updateinfo test_pool_periodic_update_sync test_pkg_mgr test_tar_ext test_pool_repository)
   (libraries
     alcotest
     bos

ocaml/tests/test_pool_repository.ml

@@ -0,0 +1,116 @@
+(*
+ * Copyright (c) Cloud Software Group, Inc.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU Lesser General Public License as published
+ * by the Free Software Foundation; version 2.1 only. with the special
+ * exception on linking described in file LICENSE.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *)
+
+module T = Test_common
+
+let test_set_remote_and_bundle_repos () =
+  let __context = T.make_test_database () in
+  let name_label = "remote" in
+  let name_description = "remote" in
+  let binary_url = "https://repo.example.com" in
+  let source_url = "https://repo-src.example.com" in
+  let gpgkey_path = "" in
+  let ref_remote =
+    Repository.introduce ~__context ~name_label ~name_description ~binary_url
+      ~source_url ~update:true ~gpgkey_path
+  in
+  let ref_bundle =
+    Repository.introduce_bundle ~__context ~name_label:"bundle"
+      ~name_description:"bundle"
+  in
+  let self = Helpers.get_pool ~__context in
+  Alcotest.check_raises "test_set_remote_and_bundle_repos"
+    Api_errors.(Server_error (bundle_repo_should_be_single_enabled, []))
+    (fun () ->
+      Xapi_pool.set_repositories ~__context ~self
+        ~value:[ref_remote; ref_bundle]
+    )
+
+let test_add_bundle_repo () =
+  let __context = T.make_test_database () in
+  let name_label = "remote" in
+  let name_description = "remote" in
+  let binary_url = "https://repo.example.com" in
+  let source_url = "https://repo-src.example.com" in
+  let gpgkey_path = "" in
+  let ref_remote =
+    Repository.introduce ~__context ~name_label ~name_description ~binary_url
+      ~source_url ~update:true ~gpgkey_path
+  in
+  let ref_bundle =
+    Repository.introduce_bundle ~__context ~name_label:"bundle"
+      ~name_description:"bundle"
+  in
+  let self = Helpers.get_pool ~__context in
+  Alcotest.check_raises "test_add_bundle_repo"
+    Api_errors.(Server_error (bundle_repo_should_be_single_enabled, []))
+    (fun () ->
+      Xapi_pool.set_repositories ~__context ~self ~value:[ref_remote] ;
+      Xapi_pool.add_repository ~__context ~self ~value:ref_bundle
+    )
+
+let test_add_remote_repo () =
+  let __context = T.make_test_database () in
+  let name_label = "remote" in
+  let name_description = "remote" in
+  let binary_url = "https://repo.example.com" in
+  let source_url = "https://repo-src.example.com" in
+  let gpgkey_path = "" in
+  let ref_remote =
+    Repository.introduce ~__context ~name_label ~name_description ~binary_url
+      ~source_url ~update:true ~gpgkey_path
+  in
+  let ref_bundle =
+    Repository.introduce_bundle ~__context ~name_label:"bundle"
+      ~name_description:"bundle"
+  in
+  let self = Helpers.get_pool ~__context in
+  Alcotest.check_raises "test_add_remote_repo"
+    Api_errors.(Server_error (bundle_repo_should_be_single_enabled, []))
+    (fun () ->
+      Xapi_pool.set_repositories ~__context ~self ~value:[ref_bundle] ;
+      Xapi_pool.add_repository ~__context ~self ~value:ref_remote
+    )
+
+let test_can_not_enable_bundle_repo_auto_sync () =
+  let __context = T.make_test_database () in
+  let ref_bundle =
+    Repository.introduce_bundle ~__context ~name_label:"bundle"
+      ~name_description:"bundle"
+  in
+  let self = Helpers.get_pool ~__context in
+  Alcotest.check_raises "test_can_not_enable_bundle_repo_auto_sync"
+    Api_errors.(Server_error (can_not_sync_updates, []))
+    (fun () ->
+      Xapi_pool.set_repositories ~__context ~self ~value:[ref_bundle] ;
+      Xapi_pool.set_update_sync_enabled ~__context ~self ~value:true
+    )
+
+let test =
+  [
+    ( "test_set_remote_and_bundle_repos"
+    , `Quick
+    , test_set_remote_and_bundle_repos
+    )
+  ; ("test_add_bundle_repo", `Quick, test_add_bundle_repo)
+  ; ("test_add_remote_repo", `Quick, test_add_remote_repo)
+  ; ( "test_can_not_enable_bundle_repo_auto_sync"
+    , `Quick
+    , test_can_not_enable_bundle_repo_auto_sync
+    )
+  ]
+
+let () =
+  Suite_init.harness_init () ;
+  Alcotest.run "Test Pool Repository suite" [("Test_pool_repository", test)]

ocaml/tests/test_repository.ml

@@ -101,7 +101,9 @@ let test_introduce_duplicate_bundle_repo () =
   in
 
   Alcotest.check_raises "test_introduce_duplicate_bundle_repo"
-    Api_errors.(Server_error (repository_already_exists, [Ref.string_of ref]))
+    Api_errors.(
+      Server_error (bundle_repository_already_exists, [Ref.string_of ref])
+    )
     (fun () ->
       Repository.introduce_bundle ~__context ~name_label:name_label_1
         ~name_description:name_description_1

ocaml/xapi-cli-server/cli_frontend.ml

@@ -3091,6 +3091,17 @@ let rec cmdtable_data : (string * cmd_spec) list =
       ; flags= []
       }
     )
+  ; ( "pool-sync-bundle"
+    , {
+        reqd= ["filename"]
+      ; optn= []
+      ; help=
+          "Upload and unpack a bundle file, after that, sync the bundle \
+           repository."
+      ; implementation= With_fd Cli_operations.pool_sync_bundle
+      ; flags= []
+      }
+    )
   ; ( "host-ha-xapi-healthcheck"
     , {
         reqd= []

ocaml/xapi-cli-server/cli_operations.ml

@@ -2462,8 +2462,7 @@ let parse_host_uuid ?(default_master = true) rpc session_id params =
     let hosts = Client.Host.get_all ~rpc ~session_id in
     let standalone = List.length hosts = 1 in
     if standalone || default_master then
-      let pool = List.hd (Client.Pool.get_all ~rpc ~session_id) in
-      Client.Pool.get_master ~rpc ~session_id ~self:pool
+      get_master ~rpc ~session_id
     else
       failwith "Required parameter not found: host-uuid"
 
@@ -3989,7 +3988,7 @@ let vm_install_real printer rpc session_id template name description params =
         , [Features.name_of_feature Features.PCI_device_for_auto_update]
         )
     in
-    let pool = List.hd (Client.Pool.get_all ~rpc ~session_id) in
+    let pool = get_pool ~rpc ~session_id in
     let policy_vendor_device_is_ok =
       not (Client.Pool.get_policy_no_vendor_device ~rpc ~session_id ~self:pool)
     in
@@ -5133,11 +5132,6 @@ let vm_cd_insert printer rpc session_id params =
 let host_careful_op op warnings fd _printer rpc session_id params =
   let uuid = List.assoc "uuid" params in
   let host = Client.Host.get_by_uuid ~rpc ~session_id ~uuid in
-  let pool = List.hd (Client.Pool.get_all ~rpc ~session_id) in
-  let _ (* unused variable 'pool_master' *) =
-    Client.Pool.get_master ~rpc ~session_id ~self:pool
-  in
-  (* if pool_master = host then failwith "Cannot forget pool master"; *)
   let force = get_bool_param params "force" in
   let go () = ignore (op ~rpc ~session_id ~self:host) in
   if force then
@@ -6605,11 +6599,11 @@ let host_disable_local_storage_caching _printer rpc session_id params =
     )
 
 let pool_enable_local_storage_caching _printer rpc session_id _params =
-  let pool = List.hd (Client.Pool.get_all ~rpc ~session_id) in
+  let pool = get_pool ~rpc ~session_id in
   Client.Pool.enable_local_storage_caching ~rpc ~session_id ~self:pool
 
 let pool_disable_local_storage_caching _printer rpc session_id _params =
-  let pool = List.hd (Client.Pool.get_all ~rpc ~session_id) in
+  let pool = get_pool ~rpc ~session_id in
   Client.Pool.disable_local_storage_caching ~rpc ~session_id ~self:pool
 
 let pool_apply_edition printer rpc session_id params =
@@ -6692,8 +6686,7 @@ let host_backup fd _printer rpc session_id params =
 let pool_dump_db fd _printer rpc session_id params =
   let filename = List.assoc "file-name" params in
   let make_command task_id =
-    let pool = List.hd (Client.Pool.get_all ~rpc ~session_id) in
-    let master = Client.Pool.get_master ~rpc ~session_id ~self:pool in
+    let master = get_master ~rpc ~session_id in
     let master_address =
       Client.Host.get_address ~rpc ~session_id ~self:master
     in
@@ -6769,6 +6762,36 @@ let pool_get_guest_secureboot_readiness printer rpc session_id params =
        (Record_util.pool_guest_secureboot_readiness_to_string result)
     )
 
+let pool_sync_bundle fd _printer rpc session_id params =
+  let filename_opt = List.assoc_opt "filename" params in
+  match filename_opt with
+  | Some filename ->
+      let make_command task_id =
+        let master = get_master ~rpc ~session_id in
+        let master_address =
+          Client.Host.get_address ~rpc ~session_id ~self:master
+        in
+        let uri =
+          Uri.(
+            make ~scheme:"http" ~host:master_address
+              ~path:Constants.put_bundle_uri
+              ~query:
+                [
+                  ("session_id", [Ref.string_of session_id])
+                ; ("task_id", [Ref.string_of task_id])
+                ]
+              ()
+            |> to_string
+          )
+        in
+        debug "%s: requesting HttpPut('%s','%s')" __FUNCTION__ filename uri ;
+        HttpPut (filename, uri)
+      in
+      ignore
+        (track_http_operation fd rpc session_id make_command "upload bundle")
+  | None ->
+      failwith "Required parameter not found: filename"
+
 let host_restore fd _printer rpc session_id params =
   let filename = List.assoc "file-name" params in
   let op _ host =

ocaml/xapi-cli-server/cli_util.ml

@@ -26,6 +26,14 @@ open Client
 
 let finally = Xapi_stdext_pervasives.Pervasiveext.finally
 
+let internal_error fmt =
+  Printf.ksprintf
+    (fun str ->
+      error "%s" str ;
+      raise Api_errors.(Server_error (internal_error, [str]))
+    )
+    fmt
+
 let log_exn_continue msg f x =
   try f x
   with e -> debug "Ignoring exception: %s while %s" (Printexc.to_string e) msg
@@ -334,3 +342,14 @@ let error_of_exn e =
 let string_of_exn exn =
   let e, l = error_of_exn exn in
   Printf.sprintf "%s: [ %s ]" e (String.concat "; " l)
+
+let get_pool ~rpc ~session_id =
+  match Client.Pool.get_all ~rpc ~session_id with
+  | [] ->
+      internal_error "Remote host does not belong to a pool."
+  | pool :: _ ->
+      pool
+
+let get_master ~rpc ~session_id =
+  let pool = get_pool ~rpc ~session_id in
+  Client.Pool.get_master ~rpc ~session_id ~self:pool