3434#include < iostream>
3535#include < fstream>
3636#include < exception>
37+ #include < future>
3738#include < uuid/uuid.h>
3839#include " xclbin.h"
3940#include " aws_dev.h"
4041
41-
42+ static std::map<std::string, size_t >index_map;
4243/*
4344 * Functions each plugin needs to provide
4445 */
@@ -59,11 +60,15 @@ int init(mpd_plugin_callbacks *cbs)
5960 syslog (LOG_INFO, " aws: no device found"
6061 return ret;
6162 }
63+ ret = AwsDev::init (index_map);
64+ if (ret)
65+ return ret;
6266 if (cbs)
6367 {
6468 // hook functions
6569 cbs->mpc_cookie = NULL ;
6670 cbs->get_remote_msd_fd = get_remote_msd_fd;
71+ cbs->mb_notify = mb_notify;
6772 cbs->mb_req .load_xclbin = awsLoadXclBin;
6873 cbs->mb_req .peer_data .get_icap_data = awsGetIcap;
6974 cbs->mb_req .peer_data .get_sensor_data = awsGetSensor;
@@ -108,6 +113,53 @@ int get_remote_msd_fd(size_t index, int* fd)
108113 return 0 ;
109114}
110115
116+ /*
117+ * callback function that is used to notify xocl the imagined xclmgmt
118+ * online/offline
119+ * Input:
120+ * index: index of the user PF
121+ * fd: mailbox file descriptor
122+ * online: online or offline
123+ * Output:
124+ * None
125+ * Return value:
126+ * 0: success
127+ * others: err code
128+ */
129+ int mb_notify (size_t index, int fd, bool online)
130+ {
131+ std::unique_ptr<sw_msg> swmsg;
132+ struct xcl_mailbox_req *mb_req = NULL ;
133+ struct xcl_mailbox_peer_state mb_conn = { 0 };
134+ size_t data_len = sizeof (struct xcl_mailbox_peer_state ) + sizeof (struct xcl_mailbox_req );
135+ pcieFunc dev (index);
136+
137+ std::vector<char > buf (data_len, 0 );
138+ mb_req = reinterpret_cast <struct xcl_mailbox_req *>(buf.data ());
139+
140+ mb_req->req = XCL_MAILBOX_REQ_MGMT_STATE;
141+ if (online)
142+ mb_conn.state_flags |= XCL_MB_STATE_ONLINE;
143+ else
144+ mb_conn.state_flags |= XCL_MB_STATE_OFFLINE;
145+ memcpy (mb_req->data , &mb_conn, sizeof (mb_conn));
146+
147+ try {
148+ swmsg = std::make_unique<sw_msg>(mb_req, data_len, 0x1234 , XCL_MB_REQ_FLAG_REQUEST);
149+ } catch (std::exception &e) {
150+ std::cout << " aws mb_notify: " what () << std::endl;
151+ throw ;
152+ }
153+
154+ struct queue_msg msg;
155+ msg.localFd = fd;
156+ msg.type = REMOTE_MSG;
157+ msg.cb = nullptr ;
158+ msg.data = std::move (swmsg);
159+
160+ return handleMsg (dev, msg);
161+ }
162+
111163/*
112164 * callback function that is used to handle MAILBOX_REQ_LOAD_XCLBIN msg
113165 *
@@ -280,6 +332,18 @@ int awsGetSubdev(size_t index, char *resp, size_t resp_len)
280332 return ret;
281333}
282334
335+ /*
336+ * Reset requires the mailbox msg return before the real reset
337+ * happens. So we run user special reset in async thread.
338+ */
339+ std::future<void > nouse; // so far we don't care the return value of reset
340+ static void awsResetDeviceAsync (size_t index)
341+ {
342+ AwsDev d (index, nullptr );
343+ if (d.isGood ()) {
344+ d.awsResetDevice ();
345+ }
346+ }
283347/*
284348 * callback function that is used to handle MAILBOX_REQ_HOT_RESET msg
285349 *
@@ -293,13 +357,9 @@ int awsGetSubdev(size_t index, char *resp, size_t resp_len)
293357 */
294358int awsResetDevice (size_t index, int *resp)
295359{
296- int ret = -1 ;
297- AwsDev d (index, nullptr );
298- if (d.isGood ()) {
299- *resp = d.awsResetDevice ();
300- ret = 0 ;
301- }
302- return ret;
360+ *resp = -ENOTSUP;
361+ nouse = std::async (std::launch::async, &awsResetDeviceAsync, index);
362+ return 0 ;
303363}
304364
305365/*
@@ -391,6 +451,16 @@ int awsReadP2pBarAddr(size_t index, const xcl_mailbox_p2p_bar_addr *addr, int *r
391451 return ret;
392452}
393453
454+ /*
455+ * On AWS F1, fpga user PF without xclbin being loaded (cleared) has different
456+ * device id (0x1042) than that of the user PF with xclbin being loaded (0xf010)
457+ * Changint the device id needs pci node remove and rescan.
458+ * fpga_pci_rescan_slot_app_pfs( mBoardNumber ) is the function used to do that.
459+ * removal of the pf requires unload the xocl driver, within mpd it is impossible.
460+ * So we assume the user already made the change by loading a default afi from cmdline
461+ * with fpga-load-local-image.
462+ * From mpd & xocl perspective, whichever device id doesn't matter.
463+ */
394464int AwsDev::awsLoadXclBin (const xclBin *buffer)
395465{
396466#ifdef INTERNAL_TESTING_FOR_AWS
@@ -519,7 +589,19 @@ bool AwsDev::isGood() {
519589
520590int AwsDev::awsUserProbe (xcl_mailbox_conn_resp *resp)
521591{
592+ #ifndef INTERNAL_TESTING_FOR_AWS
593+ fpga_slot_spec spec_array[16 ];
594+ std::memset (spec_array, 0 , sizeof (fpga_slot_spec) * 16 );
595+ if (fpga_pci_get_all_slot_specs (spec_array, 16 )) {
596+ std::cout << " ERROR: failed at fpga_pci_get_all_slot_specs"
597+ return -1 ;
598+ }
599+
600+ if (spec_array[mBoardNumber ].map [FPGA_APP_PF].device_id != AWS_UserPF_DEVICE_ID)
601+ resp->conn_flags |= XCL_MB_PEER_READY;
602+ #else
522603 resp->conn_flags |= XCL_MB_PEER_READY;
604+ #endif
523605 return 0 ;
524606}
525607
@@ -580,15 +662,18 @@ AwsDev::~AwsDev()
580662 }
581663}
582664
583- AwsDev::AwsDev (size_t index, const char *logfileName) : mBoardNumber(index)
665+ AwsDev::AwsDev (size_t index, const char *logfileName)
584666{
585667 if (logfileName != nullptr ) {
586668 mLogStream .open (logfileName);
587669 mLogStream << " FUNCTION, THREAD ID, ARG..."
588670 mLogStream << __func__ << " , " std::this_thread::get_id () << std::endl;
589671 }
590672
673+ std::string sysfs_name = pcidev::get_dev (index, true )->sysfs_name ;
674+ std::cout << " AwsDev: " " (index: " " )"
591675#ifdef INTERNAL_TESTING_FOR_AWS
676+ mBoardNumber = index;
592677 char file_name_buf[128 ];
593678 std::fill (&file_name_buf[0 ], &file_name_buf[0 ] + 128 , 0 );
594679 std::sprintf ((char *)&file_name_buf, " /dev/awsmgmt%d" mBoardNumber );
@@ -600,31 +685,13 @@ AwsDev::AwsDev(size_t index, const char *logfileName) : mBoardNumber(index)
600685
601686#else
602687 fpga_mgmt_init (); // aws-fpga version newer than 09/2019 need this
603- loadDefaultAfiIfCleared () ;
688+ mBoardNumber = index_map[sysfs_name] ;
604689 // bar0 is mapped already. seems other 2 bars are not required.
605690#endif
606691}
607692
608693// private functions
609694#ifndef INTERNAL_TESTING_FOR_AWS
610- int AwsDev::loadDefaultAfiIfCleared ( void )
611- {
612- int array_len = 16 ;
613- fpga_slot_spec spec_array[ array_len ];
614- std::memset ( spec_array, mBoardNumber , sizeof (fpga_slot_spec) * array_len );
615- fpga_pci_get_all_slot_specs ( spec_array, array_len );
616- if ( spec_array[mBoardNumber ].map [FPGA_APP_PF].device_id == AWS_UserPF_DEVICE_ID ) {
617- std::string agfi = DEFAULT_GLOBAL_AFI;
618- fpga_mgmt_load_local_image ( mBoardNumber , const_cast <char *>(agfi.c_str ()) );
619- if ( sleepUntilLoaded ( agfi ) ) {
620- std::cout << " ERROR: Sleep until load failed."
621- return -1 ;
622- }
623- fpga_pci_rescan_slot_app_pfs ( mBoardNumber );
624- }
625- return 0 ;
626- }
627-
628695int AwsDev::sleepUntilLoaded ( const std::string &afi )
629696{
630697 for ( int i = 0 ; i < 20 ; i++ ) {
0 commit comments