[feat][cache]: Asynchronous and zero-copy optimization.

Author: Wine93

conf/client.conf

@@ -208,32 +208,49 @@ data_stream.page.use_pool=true
 # disk_cache.ioring_iodepth:
 #   iodepth for io ring (works for both linux io uring and 3fs usrbio)
 block_cache.cache_store=disk
-block_cache.stage=true
-block_cache.stage_bandwidth_throttle_enable=false
-block_cache.stage_bandwidth_throttle_mb=10240
-block_cache.logging=true
-block_cache.upload_stage_workers=10
-block_cache.upload_stage_queue_size=10000
-block_cache.prefetch_workers=128
-block_cache.prefetch_queue_size=10000
+block_cache.enable_stage=true
+block_cache.enable_cache=true
+block_cache.access_logging=true
+block_cache.upload_stage_throttle_enable=false
+block_cache.upload_stage_throttle_bandwidth_mb=256
+block_cache.upload_stage_throttle_iops=100
+block_cache.prefetch_max_inflights=100
 
 disk_cache.cache_dir=/var/run/dingofs  # __DINGOADM_TEMPLATE__ /dingofs/client/data/cache __DINGOADM_TEMPLATE__
 disk_cache.cache_size_mb=102400
 disk_cache.free_space_ratio=0.1
 disk_cache.cache_expire_second=259200
 disk_cache.cleanup_expire_interval_millsecond=1000
-disk_cache.drop_page_cache=true
 disk_cache.ioring_iodepth=128
 disk_cache.ioring_blksize=1048576
 disk_cache.ioring_prefetch=true
+disk_cache.drop_page_cache=true
 
 disk_state.tick_duration_second=60
-disk_state.normal2unstable_io_error_num=3
-disk_state.unstable2normal_io_succ_num=10
+disk_state.normal2unstable_error_num=3
+disk_state.unstable2normal_succ_num=10
 disk_state.unstable2down_second=1800
 disk_state.disk_check_duration_millsecond=3000
 # }
 
+#### remote cache
+# {
+remote_cache.cache_group=
+remote_cache.load_members_interval_ms=1000
+remote_cache.mds_rpc_addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702
+remote_cache.mds_rpc_retry_total_ms=16000
+remote_cache.mds_rpc_max_timeout_ms=2000
+remote_cache.mds_rpc_timeout_ms=500
+remote_cache.mds_rpc_retry_interval_us=50000
+remote_cache.mds_rpc_max_failed_times_before_change_addr=2
+remote_cache.mds_rpc_normal_retry_times_before_trigger_wait=3
+remote_cache.mds_rpc_wait_sleep_ms=1000
+remote_cache.remote_put_rpc_timeout_ms=3000
+remote_cache.remote_range_rpc_timeout_ms=3000
+remote_cache.remote_cache_rpc_timeout_ms=3000
+remote_cache.remote_prefetch_rpc_timeout_ms=3000
+# }
+
 #### s3
 # this is for test. if s3.fakeS3=true, all data will be discarded
 s3.fakeS3=false
@@ -280,7 +297,7 @@ s3.enableTelemetry=false
 
 s3.use_crt_client=false
 # this only work when use_crt_client is false
-s3.use_thread_pool=true 
+s3.use_thread_pool=true
 # this only work when use_crt_client is false and use_thread_pool is true
 s3.async_thread_num_in_thread_pool=256
 

confv2/cache-bench.conf

@@ -0,0 +1,62 @@
+############################# global
+--logdir="/tmp/dingofs-cache-bench"
+--loglevel=0
+
+############################# benchmark
+--opt=put
+--threads=1
+--op_blksize=4194304
+--op_blocks=1
+--put_writeback=true
+--range_retrive=false
+--fsid=1
+--ino=0
+--s3_ak=
+--s3_sk=
+--s3_endpoint=
+--s3_bucket=
+--stat_interval_s=3
+
+############################# block cache
+--cache_store=disk
+--enable_stage=true
+--enable_cache=true
+--cache_access_logging=true
+--upload_stage_throttle_enable=false
+--upload_stage_throttle_bandwidth_mb=256
+--upload_stage_throttle_iops=100
+--prefetch_max_inflights=100
+
+############################# disk cache
+--cache_dir=/tmp/dingofs-cache
+--cache_size_mb=10240
+--free_space_ratio=0.1
+--cache_expire_s=259200
+--cleanup_expire_interval_ms=1000
+--ioring_blksize=1048576
+--ioring_iodepth=128
+--ioring_prefetch=true
+--drop_page_cache=true
+
+############################# disk state
+--state_tick_duration_s=60
+--state_normal2unstable_error_num=3
+--state_unstable2normal_succ_num=10
+--state_unstable2down_s=1800
+--check_disk_state_duration_ms=3000
+
+############################# remote cache
+--group_name=
+--load_members_interval_ms=1000
+--mds_rpc_addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702
+--mds_rpc_retry_total_ms=16000
+--mds_rpc_max_timeout_ms=2000
+--mds_rpc_timeout_ms=500
+--mds_rpc_retry_interval_us=50000
+--mds_rpc_max_failed_times_before_change_addr=2
+--mds_rpc_normal_retry_times_before_trigger_wait=3
+--mds_rpc_wait_sleep_ms=1000
+--remote_put_rpc_timeout_ms=3000
+--remote_range_rpc_timeout_ms=3000
+--remote_cache_rpc_timeout_ms=3000
+--remote_prefetch_rpc_timeout_ms=3000

confv2/dingo-cache.conf

@@ -0,0 +1,49 @@
+############################# global
+--logdir=/tmp/dingofs-logs
+--loglevel=0
+
+############################# cache group node
+--group_nane=default
+--listen_ip=127.0.0.1
+--listen_port=20000
+--group_weight=100
+--max_range_size_kb=256
+--metadata_filepath="/tmp/cache_group_metadata"
+--send_heartbeat_interval_ms=1000
+--mds_rpc_addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702
+--mds_rpc_retry_total_ms=16000
+--mds_rpc_max_timeout_ms=2000
+--mds_rpc_timeout_ms=500
+--mds_rpc_retry_interval_us=50000
+--mds_rpc_max_failed_times_before_change_addr=2
+--mds_rpc_normal_retry_times_before_trigger_wait=3
+--mds_rpc_wait_sleep_ms=1000
+
+############################# block cache
+--cache_store=disk
+--enable_stage=true
+--enable_cache=true
+--cache_access_logging=true
+--upload_stage_throttle_enable=false
+--upload_stage_throttle_bandwidth_mb=256
+--upload_stage_throttle_iops=100
+--prefetch_max_inflights=100
+--upload_stage_max_inflights=128
+
+############################# disk cache
+--cache_dir=/tmp/dingofs-cache
+--cache_size_mb=10240
+--free_space_ratio=0.1
+--cache_expire_s=259200
+--cleanup_expire_interval_ms=1000
+--ioring_blksize=1048576
+--ioring_iodepth=128
+--ioring_prefetch=true
+--drop_page_cache=true
+
+############################# disk state
+--state_tick_duration_s=60
+--state_normal2unstable_error_num=3
+--state_unstable2normal_succ_num=10
+--state_unstable2down_s=1800
+--check_disk_state_duration_ms=3000

confv2/dingo-cache.gflags

@@ -0,0 +1,115 @@
+############################# global
+--access_logging=true
+--bthread_worker_num=0
+
+############################# fuse
+--fuse_conn_info_want_splice_move=false
+--fuse_conn_info_want_splice_read=false
+--fuse_conn_info_want_splice_write=false
+--fuse_conn_info_want_auto_inval_data=true
+--fuse_file_info_direct_io=false
+--fuse_file_info_keep_cache=true
+
+############################# vfs
+vfs_writeback_suffix=
+vfs_max_name_length=255
+vfs_disable_xattr=true
+
+vfs_kernel_cache_attr_timeout_s=30
+vfs_kernel_cache_dir_attr_timeout_s=30
+vfs_kernel_cache_entry_timeout_s=30
+vfs_kernel_cache_dir_entry_timeout_s=30
+vfs_lookup_cache_negative_timeout_s=0
+vfs_lookup_cache_min_uses=1
+vfs_lookup_cache_lru_size=100000
+vfs_dir_cache_lru_size=5000000
+vfs_attr_watcher_lru_size=5000000
+
+vfs_throttle_avg_write_bytes=0
+vfs_throttle_burst_write_bytes=0
+vfs_throttle_burst_write_bytes_s=180
+vfs_throttle_avg_write_iops=0
+vfs_throttle_burst_write_iops=0
+vfs_throttle_burst_write_iops_s=180
+vfs_throttle_avg_read_bytes=0
+vfs_throttle_burst_read_bytes=0
+vfs_throttle_burst_read_bytes_s=180
+vfs_throttle_avg_read_iops=0
+vfs_throttle_burst_read_iops=0
+vfs_throttle_burst_read_iops_s=180
+
+############################# meta
+--meta_v2_mds_rpc_timeout_ms=2000
+--meta_v2_mds_rpc_retry_times=3
+--meta_v2_client_send_request_retry_times=3
+
+############################# data
+--data_background_flush_interval_ms=1000
+--data_background_flush_trigger_force_memory_ratio=0.90
+--data_flush_file_max_size_mb=1024
+
+############################# block cache
+--cache_store=disk
+--enable_stage=true
+--enable_cache=true
+--cache_access_logging=true
+--upload_stage_throttle_enable=false
+--upload_stage_throttle_bandwidth_mb=256
+--upload_stage_throttle_iops=100
+--prefetch_max_inflights=100
+--upload_stage_max_inflights=128
+
+############################# disk cache
+--cache_dir=/tmp/dingofs-cache
+--cache_size_mb=10240
+--free_space_ratio=0.1
+--cache_expire_s=259200
+--cleanup_expire_interval_ms=1000
+--ioring_blksize=1048576
+--ioring_iodepth=128
+--ioring_prefetch=true
+--drop_page_cache=true
+
+############################# disk state
+--state_tick_duration_s=60
+--state_normal2unstable_error_num=3
+--state_unstable2normal_succ_num=10
+--state_unstable2down_s=1800
+--check_disk_state_duration_ms=3000
+
+############################# remote cache
+--group_name=
+--load_members_interval_ms=1000
+--mds_rpc_addrs=127.0.0.1:6700,127.0.0.1:6701,127.0.0.1:6702
+--mds_rpc_retry_total_ms=16000
+--mds_rpc_max_timeout_ms=2000
+--mds_rpc_timeout_ms=500
+--mds_rpc_retry_interval_us=50000
+--mds_rpc_max_failed_times_before_change_addr=2
+--mds_rpc_normal_retry_times_before_trigger_wait=3
+--mds_rpc_wait_sleep_ms=1000
+--remote_put_rpc_timeout_ms=3000
+--remote_range_rpc_timeout_ms=3000
+--remote_cache_rpc_timeout_ms=3000
+--remote_prefetch_rpc_timeout_ms=3000
+
+############################# block accesser
+--s3_telemetry_enable=false
+
+--s3_request_region=us-east-1
+--s3_request_use_virtual_addressing=false
+--s3_request_verify_ssl=false
+--s3_request_max_connections=32
+--s3_request_connect_timeout_ms=60000
+--s3_request_request_timeout_ms=10000
+--s3_request_use_crt_client=false
+--s3_request_use_thread_pool=true
+--s3_request_async_thread_num=4
+--s3_request_max_async_request_inflight_bytes=0
+
+--s3_throttle_iops_total_limit=0
+--s3_throttle_iops_read_limit=0
+--s3_throttle_iops_write_limit=0
+--s3_throttle_bps_total_mb=0
+--s3_throttle_bps_read_mb=0
+--s3_throttle_bps_write_mb=0