Merge pull request #292 from FedeParola/controller-direct

Send packets to controller from the running program

Author: frisso

Documentation/developers/dataplane.rst

@@ -18,11 +18,18 @@ Polycube architecture adds a wrapper around the user's code, this wrapper calls
 
 - **pcn_pkt_drop(struct __sk_buff *skb, struct pkt_metadata *md);**: drops the packet. It is the same that just returning `RX_DROP`. [Example](services/pcn-helloworld/src/Helloworld_dp.h#L78)
 
-- **pcn_pkt_controller(struct __sk_buff *skb, struct pkt_metadata *md, u16 reason)**: sends the packet to the control path controller. Reason can be used to indicate why the packet is being sent to the custom code running in the control path. If there is not any reason `RX_CONTROLLER` could be directly returned. [Example](services/pcn-helloworld/src/Helloworld_dp.h#L82)
+- **pcn_pkt_redirect_ns(struct __sk_buff *skb, struct pkt_metadata *md, u16 port)**: (it is only available for shadow services) sends the packet to the namespace as if it came from the port indicated as parameter
 
-- **pcn_pkt_controller_with_metadata(struct __sk_buff *skb, struct pkt_metadata *md, u16 reason, u32 metadata[3])**: Sends the packet to the custom code running in the control path. In addition to the reason the user can also send some additional medatada.
+Processing packets in the slowpath
+**********************************
 
-- **pcn_pkt_redirect_ns(struct __sk_buff *skb, struct pkt_metadata *md, u16 port)**: (it is only available for shadow services) sends the packet to the namespace as if it came from the port indicated as parameter
+A copy of the packet can be sent to the controller to be processed by the slowpath using the following helpers:
+
+- **pcn_pkt_controller(struct __sk_buff *skb, struct pkt_metadata *md, u16 reason)**: sends a copy of the packet to the controller. Reason can be used to indicate why the packet is being sent to the custom code running in the control path. [Example](services/pcn-helloworld/src/Helloworld_dp.h#L82)
+ 
+- **pcn_pkt_controller_with_metadata(struct __sk_buff *skb, struct pkt_metadata *md, u16 reason, u32 metadata[3])**: sends a copy of the packet to the custom code running in the control path. In addition to the reason the user can also send some additional metadata.
+ 
+The packet will be processed by the ``packet_in`` method of the controller.
 
 Checksum calculation
 ********************

Documentation/services/pcn-packetcapture/packetcapture.rst

@@ -192,5 +192,5 @@ As a example, we list here is the generated C code for the filter ``icmp``:
          } else {
            goto L5;
          }
-    L4:	 return pcn_pkt_controller(ctx, md, reason);
+    L4:  pcn_pkt_controller(ctx, md, reason);
     L5:	 return RX_OK;

src/polycubed/src/base_cube.cpp

@@ -408,7 +408,6 @@ enum {
   RX_REDIRECT,
   RX_DROP,
   RX_RECIRCULATE,
-  RX_CONTROLLER,
   RX_ERROR,
 };
 

src/polycubed/src/controller.cpp

@@ -38,65 +38,9 @@ std::map<int, const packet_in_cb &> Controller::cbs_;
 
 std::mutex Controller::cbs_mutex_;
 
-// Sends the packet to the controller
-const std::string CTRL_TC_TX = R"(
-struct metadata {
-  u16 cube_id;
-  u16 port_id;
-  u32 packet_len;
-  u32 traffic_class;
-  u16 reason;
-  u32 md[3];  // generic metadata
-} __attribute__((packed));
-
-BPF_PERF_OUTPUT(controller);
-
-int controller_module_tx(struct __sk_buff *ctx) {
-  pcn_log(ctx, LOG_TRACE, "[tc-encapsulator]: to controller");
-  // If the packet is tagged add the tagged in the packet itself, otherwise it
-  // will be lost
-  if (ctx->vlan_present) {
-    volatile __u32 vlan_tci = ctx->vlan_tci;
-    volatile __u32 vlan_proto = ctx->vlan_proto;
-    bpf_skb_vlan_push(ctx, vlan_proto, vlan_tci);
-  }
-
-  volatile u32 x; // volatile to avoid verifier error on kernels < 4.10
-  x = ctx->cb[0];
-  u16 in_port = x >> 16;
-  u16 module_index = x & 0x7fff;
-  u16 pass_to_stack = x & 0x8000;
-
-  x = ctx->cb[1];
-  u16 reason = x & 0xffff;
-
-  struct metadata md = {0};
-  md.cube_id = module_index;
-  md.port_id = in_port;
-  md.packet_len = ctx->len;
-  md.traffic_class = ctx->mark;
-  md.reason = reason;
-
-  x = ctx->cb[2];
-  md.md[0] = x;
-  x = ctx->cb[3];
-  md.md[1] = x;
-  x = ctx->cb[4];
-  md.md[2] = x;
-
-  int r = controller.perf_submit_skb(ctx, ctx->len, &md, sizeof(md));
-  if (r != 0) {
-    pcn_log(ctx, LOG_ERR, "[tc-encapsulator]: perf_submit_skb error: %d", r);
-  }
-  if (pass_to_stack) {
-    pcn_log(ctx, LOG_DEBUG, "[tc-encapsulator]: passing to stack");
-    return 0;
-  }
-  return 7;
-ERROR:
-  pcn_log(ctx, LOG_ERR, "[tc-encapsulator]: unknown error");
-  return 7;	//TODO: check code
-}
+// Perf buffer used to send packets to the controller
+const std::string CTRL_PERF_BUFFER = R"(
+BPF_TABLE_PUBLIC("perf_output", int, __u32, _BUFFER_NAME, 0);
 )";
 
 // Receives packet from controller and forwards it to the Cube
@@ -168,66 +112,6 @@ int controller_module_rx(struct __sk_buff *ctx) {
 }
 )";
 
-const std::string CTRL_XDP_TX = R"(
-#include <bcc/helpers.h>
-#include <bcc/proto.h>
-
-#include <uapi/linux/bpf.h>
-#include <uapi/linux/if_ether.h>
-
-struct pkt_metadata {
-  u16 cube_id;
-  u16 in_port;
-  u32 packet_len;
-  u32 traffic_class;
-
-  // used to send data to controller
-  u16 reason;
-  u32 md[3];  // generic metadata
-} __attribute__((packed));
-
-BPF_PERF_OUTPUT(controller);
-BPF_TABLE_PUBLIC("percpu_array", u32, struct pkt_metadata, port_md, 1);
-
-int controller_module_tx(struct xdp_md *ctx) {
-  pcn_log(ctx, LOG_TRACE, "[xdp-encapsulator]: to controller");
-  void* data_end = (void*)(long)ctx->data_end;
-  void* data = (void*)(long)ctx->data;
-  u32 inport_key = 0;
-  struct pkt_metadata *int_md;
-  struct pkt_metadata md = {0};
-  volatile u32 x;
-
-  int_md = port_md.lookup(&inport_key);
-  if (int_md == NULL) {
-    goto ERROR;
-  }
-
-  md.cube_id = int_md->cube_id;
-  md.in_port = int_md->in_port;
-  md.packet_len = (u32)(data_end - data);
-  md.traffic_class = int_md->traffic_class;
-  md.reason = int_md->reason;
-
-  x = int_md->md[0];
-  md.md[0] = x;
-  x = int_md->md[1];
-  md.md[1] = x;
-  x = int_md->md[2];
-  md.md[2] = x;
-
-  int r = controller.perf_submit_skb(ctx, md.packet_len, &md, sizeof(md));
-  if (r != 0) {
-    pcn_log(ctx, LOG_ERR, "[xdp-encapsulator]: perf_submit_skb error: %d", r);
-  }
-
-  return BPF_REDIRECT;
-ERROR:
-  pcn_log(ctx, LOG_ERR, "[xdp-encapsulator]: unknown error");
-  return XDP_DROP;
-}
-)";
-
 // Receives packet from controller and forwards it to the CubeXDP
 const std::string CTRL_XDP_RX = R"(
 #include <bcc/helpers.h>
@@ -255,7 +139,7 @@ struct pkt_metadata {
 } __attribute__((packed));
 
 BPF_TABLE("extern", int, int, xdp_nodes, _POLYCUBE_MAX_NODES);
-BPF_TABLE("extern", u32, struct pkt_metadata, port_md, 1);
+BPF_TABLE_PUBLIC("percpu_array", u32, struct pkt_metadata, port_md, 1);
 BPF_TABLE("array", u32, struct xdp_metadata, md_map_rx, MD_MAP_SIZE);
 BPF_TABLE("array", u32, u32, xdp_index_map_rx, 1);
 
@@ -324,38 +208,34 @@ void Controller::call_back_proxy(void *cb_cookie, void *data, int data_size) {
 }
 
 Controller &Controller::get_tc_instance() {
-  PatchPanel::get_tc_instance();
-  static Controller instance(CTRL_TC_TX, CTRL_TC_RX, BPF_PROG_TYPE_SCHED_CLS);
+  static Controller instance("controller_tc", CTRL_TC_RX,
+                             BPF_PROG_TYPE_SCHED_CLS);
   static bool initialized = false;
   if (!initialized) {
     Netlink::getInstance().attach_to_tc(instance.iface_->getName(),
                                         instance.fd_rx_);
-    PatchPanel::get_tc_instance().add(instance.get_fd(),
-                                      PatchPanel::_POLYCUBE_MAX_NODES - 1);
     initialized = true;
   }
   return instance;
 }
 
 Controller &Controller::get_xdp_instance() {
-  PatchPanel::get_xdp_instance();
-  static Controller instance(CTRL_XDP_TX, CTRL_XDP_RX, BPF_PROG_TYPE_XDP);
+  static Controller instance("controller_xdp", CTRL_XDP_RX, BPF_PROG_TYPE_XDP);
   static bool initialized = false;
   if (!initialized) {
     int attach_flags = 0;
     attach_flags |= 2 << 0;
     Netlink::getInstance().attach_to_xdp(instance.iface_->getName(),
                                          instance.fd_rx_, attach_flags);
-    PatchPanel::get_xdp_instance().add(instance.get_fd(),
-                                       PatchPanel::_POLYCUBE_MAX_NODES - 1);
     initialized = true;
   }
   return instance;
 }
 
-Controller::Controller(const std::string &tx_code, const std::string &rx_code,
-                       enum bpf_prog_type type)
-    : ctrl_rx_md_index_(0),
+Controller::Controller(const std::string &buffer_name,
+                       const std::string &rx_code, enum bpf_prog_type type)
+    : buffer_name_(buffer_name),
+      ctrl_rx_md_index_(0),
       logger(spdlog::get("polycubed")),
       id_(PatchPanel::_POLYCUBE_MAX_NODES - 1) {
   ebpf::StatusTuple res(0);
@@ -382,23 +262,18 @@ Controller::Controller(const std::string &tx_code, const std::string &rx_code,
 
   datapath_log.register_cb(get_id(), handle_log_msg);
 
-  std::string tx_code_(datapath_log.parse_log(tx_code));
-  std::string rx_code_(datapath_log.parse_log(rx_code));
-
-  res = tx_module_.init(tx_code_, flags);
-  if (res.code() != 0) {
-    logger->error("cannot init ctrl_tx: {0}", res.msg());
-    throw BPFError("cannot init controller tx program");
-  }
-
-  res = tx_module_.load_func("controller_module_tx", type, fd_tx_);
+  // Load perf buffer
+  std::string buffer_code(CTRL_PERF_BUFFER);
+  buffer_code.replace(buffer_code.find("_BUFFER_NAME"), 12, buffer_name);
+ 
+  res = buffer_module_.init(buffer_code, flags);
   if (res.code() != 0) {
-    logger->error("cannot load ctrl_tx: {0}", res.msg());
-    throw BPFError("cannot load controller_module_tx");
+    logger->error("cannot init ctrl perf buffer: {0}", res.msg());
+    throw BPFError("cannot init controller perf buffer");
   }
 
-  res =
-      tx_module_.open_perf_buffer("controller", call_back_proxy, nullptr, this);
+  res = buffer_module_.open_perf_buffer(buffer_name_, call_back_proxy, nullptr,
+                                        this);
   if (res.code() != 0) {
     logger->error("cannot open perf ring buffer for controller: {0}",
                   res.msg());
@@ -411,7 +286,7 @@ Controller::Controller(const std::string &tx_code, const std::string &rx_code,
   iface_->setMTU(9000);
   iface_->up();
 
-  res = rx_module_.init(rx_code_, flags);
+  res = rx_module_.init(datapath_log.parse_log(rx_code), flags);
   if (res.code() != 0) {
     logger->error("cannot init ctrl_rx: {0}", res.msg());
     throw BPFError("cannot init controller rx program");
@@ -483,7 +358,7 @@ void Controller::start() {
   auto f = [&]() -> void {
     stop_ = false;
     while (!stop_) {
-      tx_module_.poll_perf_buffer("controller", 500);
+      buffer_module_.poll_perf_buffer(buffer_name_, 500);
     }
 
     // TODO: this causes a segmentation fault

src/polycubed/src/controller.h

@@ -93,7 +93,8 @@ class Controller {
 
   bool stop_;
 
-  ebpf::BPF tx_module_;
+  std::string buffer_name_;
+  ebpf::BPF buffer_module_;
   ebpf::BPF rx_module_;
   int fd_tx_;
   int fd_rx_;

src/polycubed/src/cube_tc.cpp

@@ -198,13 +198,16 @@ BPF_TABLE("extern", int, int, nodes, _POLYCUBE_MAX_NODES);
 BPF_PERF_OUTPUT(span_slowpath);
 #endif
 
-static __always_inline
-int to_controller(struct CTXTYPE *skb, u16 reason) {
-  skb->cb[1] = reason;
-  nodes.call(skb, CONTROLLER_MODULE_INDEX);
-  //bpf_trace_printk("to controller miss\n");
-  return TC_ACT_OK;
-}
+struct controller_table_t {
+  int key;
+  u32 leaf;
+  /* map.perf_submit(ctx, data, data_size) */
+  int (*perf_submit) (void *, void *, u32);
+  int (*perf_submit_skb) (void *, u32, void *, u32);
+  u32 data[0];
+};
+__attribute__((section("maps/extern")))
+struct controller_table_t controller_tc;
 
 #if defined(SHADOW) && defined(SPAN)
 static __always_inline
@@ -219,33 +222,14 @@ int pcn_pkt_drop(struct CTXTYPE *skb, struct pkt_metadata *md) {
   return RX_DROP;
 }
 
-static __always_inline
-int pcn_pkt_controller(struct CTXTYPE *skb, struct pkt_metadata *md,
-                       u16 reason) {
-  md->reason = reason;
-  return RX_CONTROLLER;
-}
-
-static __always_inline
-int pcn_pkt_controller_with_metadata_stack(struct CTXTYPE *skb,
-                                           struct pkt_metadata *md,
-                                           u16 reason,
-                                           u32 metadata[3]) {
-  skb->cb[0] |= 0x8000;
-  skb->cb[2] = metadata[0];
-  skb->cb[3] = metadata[1];
-  skb->cb[4] = metadata[2];
-  return pcn_pkt_controller(skb, md, reason);
-}
-
 static __always_inline
 int pcn_pkt_controller_with_metadata(struct CTXTYPE *skb,
                                      struct pkt_metadata *md,
                                      u16 reason,
                                      u32 metadata[3]) {
-  skb->cb[2] = metadata[0];
-  skb->cb[3] = metadata[1];
-  skb->cb[4] = metadata[2];
+  md->md[0] = metadata[0];
+  md->md[1] = metadata[1];
+  md->md[2] = metadata[2];
   return pcn_pkt_controller(skb, md, reason);
 }
 
@@ -377,9 +361,6 @@ int handle_rx_wrapper(struct CTXTYPE *skb) {
       return forward(skb, md.reason);
     case RX_DROP:
       return TC_ACT_SHOT;
-    case RX_CONTROLLER:
-      return to_controller(skb, md.reason);
-
     case RX_OK: {
 #if POLYCUBE_PROGRAM_TYPE == 1  // EGRESS
       // If there is another egress program call it, otherwise let the packet
@@ -431,6 +412,24 @@ int pcn_pkt_redirect_ns(struct CTXTYPE *skb,
 #endif
   return TC_ACT_SHOT;
 }
+
+static __always_inline
+int pcn_pkt_controller(struct CTXTYPE *skb, struct pkt_metadata *md,
+                       u16 reason) {
+  // If the packet is tagged add the tagged in the packet itself, otherwise it
+  // will be lost
+  if (skb->vlan_present) {
+    volatile __u32 vlan_tci = skb->vlan_tci;
+    volatile __u32 vlan_proto = skb->vlan_proto;
+    bpf_skb_vlan_push(skb, vlan_proto, vlan_tci);
+  }
+
+  md->cube_id = CUBE_ID;
+  md->packet_len = skb->len;
+  md->reason = reason;
+
+  return controller_tc.perf_submit_skb(skb, skb->len, md, sizeof(*md));
+}
 )";
 
 }  // namespace polycubed