From 63bfc3ba0443f071c201369e36b72bcd6cb376fd Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Wed, 21 Aug 2024 12:00:03 +0530
Subject: [PATCH 01/14] phase-1 docs for layer2 support in CAPP: version 1

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 382 ++++++++++++++++++++++++++++++++++
 1 file changed, 382 insertions(+)
 create mode 100644 docs/layer2/layer2_support.md

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
new file mode 100644
index 00000000..33b34f71
--- /dev/null
+++ b/docs/layer2/layer2_support.md
@@ -0,0 +1,382 @@
+Motivation/Abstract
+===================
+
+By default all servers that are created on Equinix Metal via Cluster API have Layer 3 networking and there is no option when provisioning Equinix Metal cluster to specify type of networking for instances or to create additional L2 interfaces with specific local IP addresses and VLAN.
+
+To solve this, CAPP should provide options to specify:
+
+-   Network type (L2/L3/Hybrid)
+
+-   Creating network interfaces with specific VLAN
+
+-   IP address range for L2 interfaces
+
+* * * * *
+
+Limitations
+===========
+
+CAPP managed clusters running without internet connections would need to be able to pull images from a repository also in the layer2, or they would need a bastion host that acts as a gateway and NAT. This isn't supported today, so complete Layer2 will not be supported in the initial phases of the feature.
+
+* * * * *
+
+Background
+==========
+
+User stories
+------------
+
+As a  cluster administrator
+
+- I want  to configure L2 interfaces and define my own IP Address range so that  machines are able to communicate over layer2 VLAN.
+
+* * * * *
+
+Goals (phase-1)
+===============
+
+-   For the phase-1 of supporting Layer2 functionalities in CAPP, it will add support for [Hybrid Bonded](https://deploy.equinix.com/developers/docs/metal/layer2-networking/hybrid-bonded-mode/) and [Hybrid Unbonded](https://deploy.equinix.com/developers/docs/metal/layer2-networking/hybrid-unbonded-mode/) mode.
+
+-   CAPP will be able to attach ports to specified VLAN.
+
+-   CAPP will be able to create a new VLAN if required in the specified project and metro.
+
+-   CAPP will be able to create VRF (Virtual Routing & Forwarding) and IP Reservations as specified in the spec.
+
+-   CAPP will be able to configure networking at the OS level of a metal node.
+
+Non-Goals
+===============
+
+-   Complete layer2 will not be supported in the initial phases.
+
+-   IPAM Provider will be supported in phase-2
+
+Proposal Design/Approach
+========================
+
+* * * * *
+
+### PacketCluster
+
+The idea is to introduce options to specify VLANs and VRFs configurations as a part of ClusterSpec.
+
+```go
+type  PacketClusterSpec struct {
+  ...
+
+// VLANConfig represents the configuration for layer2 VLAN.
+ VLANConfig *VLANConfig            `json:"vlanConfig,omitempty"`
+
+// VRFConfig represents the configuration for Virtual  Router and Forwarding (VRF)
+ VRFConfig *VRFConfig              `json:"vrfConfig,omitempty"`
+}
+```
+
+// VLANConfig represents the configuration for an existing or new VLAN
+```go
+type VLANConfig struct {
+ // VLANID is the ID of the VLAN. It can be an existing VLAN or a new one to be created.
+  VLANID string `json:"vlanID"`
+
+ // Description of the VLAN
+ // +optional
+  Description string `json:"description,omitempty"`
+
+ // CreateNew indicates whether a new VLAN needs to be created
+ // +optional
+  CreateNew bool `json:"createNew,omitempty"`
+}
+```
+
+-   For an existing VLAN
+```json
+{
+"vlanID": "existing-vlan-id",
+"description": "Existing VLAN",
+"createNew": false
+}
+```
+
+-   For a new VLAN:
+
+```json
+{
+"vlanID": "new-vlan-id",
+"description": "New VLAN to be created",
+"createNew": true
+}
+```
+
+* * * * *
+
+
+// VRFConfig represents the configuration for an existing or new VRF
+
+```go
+type VRFConfig struct {
+ // Name of the VRF. It can be an existing VRF or a new one to be created.
+  Name string `json:"name,omitempty"`
+
+ // Description of the VRF
+ // +optional
+  Description string `json:"description,omitempty"`
+
+ // IP ranges that are allowed to access and communicate with this virtual router.
+  AllowedIPRanges []string `json:"allowedIPRanges,omitempty"`
+
+ // CreateNew indicates whether a new VRF needs to be created\
+ // +optional
+  CreateNew bool `json:"createNew,omitempty"`
+}
+```
+
+- For an existing VRF:
+
+```json
+{
+"name": "Existing VRF",
+"description": "Existing VRF description",
+"allowedIPRanges": ["192.168.1.0/21"],
+"createNew": false
+}
+```
+
+- For a new VRF:
+```json
+{
+"name": "New VRF",
+"description": "New VRF description",
+  "allowedIPRanges": ["10.0.0.0/24"],
+"createNew": true
+}
+```
+
+Users can specify different options for CAPP Controllers to create a new VLAN/VRF or to provide existing configuration details. Below is an example YAML configuration illustrating this:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1\
+kind: PacketCluster
+metadata:
+  name: capi-quickstart
+  namespace: default
+spec:
+  metro: da
+  projectID:  7f713f3e-fe14-4386-a2fb-f22c29ba685e\
+  vipManager: CPEM
+  vlanConfig:
+ "vlanID": "existing-vlan-id",
+"description": "Existing VLAN",
+"createNew": false
+  vrfConfig: 
+    "name": "New VRF",
+    "description": "New VRF description",
+    "allowedIPRanges": ["10.0.0.0/24"],
+    "createNew": true
+```
+
+In the above example, vlanConfig and vrfConfig are used to create a new VLAN and a new VRF with specific configurations, respectively.
+
+### Infrastructure Setup and Readiness
+
+* * * * *
+
+The PacketCluster Controller is responsible for setting up infrastructure. Once the setup is successfully completed, it marks itself as "Ready" and sets the condition NetworkInfrastructureReadyCondition to true. The controller will continue to follow this setup process and call Equinix Metal APIs to create or get VLANs and VRFs as needed before marking the infrastructure as Ready.
+
+This approach also helps avoid potential race conditions between the execution of user-data scripts and the application of port configurations at the API level. By ensuring the network configurations are established through the controller before any user-data scripts run, we can maintain a consistent and reliable infrastructure setup process.
+
+### APIs
+
+* * * * *
+
+1.  Create a new VLAN
+
+```
+curl -X POST
+-H "Content-Type: application/json"
+-H "X-Auth-Token: <API_TOKEN>"
+"https://api.equinix.com/metal/v1/projects/{id}/virtual-networks"
+-d '{
+ "vxlan": <integer>,
+ "description": "<string>",
+ "metro": "<string>"
+ }'
+```
+
+2.  Create a new VRF
+```
+curl -X POST
+-H "Content-Type: application/json"
+-H "X-Auth-Token: <API_TOKEN>"
+"https://api.equinix.com/metal/v1/projects/{id}/vrfs"
+-d '{
+ "name": "<string>",
+ "description": "<string>",
+ "metro": "<metro_slug>",
+ "ip_ranges": [
+ "<cidr_address>"
+    ],
+ "local_asn": <integer>
+}'
+```
+
+### PacketMachineTemplate
+
+
+* * * * *
+
+The idea is to introduce a new field Ports under spec which will contain different networking specifications of networking ports for a Equinix Metal Machine. This could look like the following:
+
+```go
+// PacketMachineSpec defines the desired state of  PacketMachine.
+type  PacketMachineSpec struct {
+  ..
+// List of Port  Configurations on each Packet  Machine\
+  // +optional\
+ Ports []Port `json:"ports"`
+}
+```
+
+```go
+type Port struct {
+ // name of the port e.g bond0
+    Name string `json:"name"`
+ // port bonded or not - by default true
+    Bonded bool `json:"bonded,omitempty"`
+ // convert port to layer 2. is false by default on new devices. changes result in /ports/id/convert/layer-[2|3] API calls\
+    Layer2 bool `json:"layer2"`
+
+    IPAddresses []IPAddress `json:"ip_addresses,omitempty"`
+}
+// IPAddress represents an IP address configuration\
+type IPAddress struct {
+    Type string `json:"type"`
+
+    // VRF used to create the IP Address Reservation for this cluster\
+    VRFName string `json:"vrfName,omitempty"`
+
+    // IPAddressReservation to reserve for these cluster nodes.
+    IPAddressReservation string `json:"ipAddressReservation"`
+
+    // VLAN Id to join
+    VLANId string `json:"vlanId,omitempty"`
+}
+```
+
+Example:
+
+```yaml
+apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
+kind: PacketMachineTemplate\
+metadata:
+    name: example-packet-machine-template
+spec:
+  template:
+    spec:
+        facility: ny5
+        metro: ny
+        plan: c3.small.x86
+        billingCycle: hourly
+        project: your-packet-project-id
+        sshKeys:
+        -  ssh-rsa  AAAAB3...your-public-key...
+        operatingSystem: ubuntu_20_04
+        ports: # network_ports in the API naming\
+        - bond0:
+            bonded: true  # default\
+            layer2: false
+            ip_addresses:
+            - type: vrf
+              vrfName: "vrf-01"
+              ipAddressReservation: "192.168.2.0/24"
+              vlanId: 1000
+        - eth2: # unbonded eth ports can use most of the same attributes bond ports can use\
+            layer2: true
+```
+
+The CAPP Operators while provisioning the metal node would:
+
+1.  Call the Equinix Metal API to attach Ports to the VLAN 
+
+```
+curl -X POST
+-H "Content-Type: application/json"
+-H "X-Auth-Token: <API_TOKEN> "
+"https://api.equinix.com/metal/v1/ports/{id}/assign"
+-d '{
+    "vnid": "<vlan_ID>"
+    }'
+```
+
+1.  Once all the ports are attached to the respective VLANs, CAPP needs to configure the networking on the server's operating system.
+
+2.  It will then GET the VRF and update it to reserve the IPAddressReservation.
+
+```
+curl -X POST
+-H 'Content-Type: application/json'
+-H "X-Auth-Token: <API_TOKEN>"
+"https://api.equinix.com/metal/v1/projects/{id}/ips"
+-d '{
+    "cidr": <integer>,
+    "network": "<ip_address>",
+    "type": "vrf",
+    "vrf_id": "<UUID>"
+}'
+```
+
+1.  For each Machine, CAPP needs to pick an IP Address from the above range and assign it at the OS Level. In Phase 1, it will be done by CAPP itself (explained below) and later in phase-2, an IPAM Provider will be implemented as per CAPI standards.
+
+2.  Once it receives an IP Address that is non-conflicting, It will create subinterfaces to handle the tagged traffic over the VLAN and assign the IP Address on that sub-interface. Hybrid Bonded mode does not support untagged VLAN traffic or setting a Native VLAN.
+
+3.  In order to configure networking on the OS, user-data and cloud-config format could be utilized as shown below in the example.
+
+Example:
+
+```sh
+#cloud-config\
+write_files:\
+  - path: /etc/network/interfaces
+
+    append: true\
+    content: |
+
+      auto bond0.${VLAN_ID_0}\
+        iface bond0.${VLAN_ID_0} inet static
+
+        pre-up sleep 5\
+address 192.168.2.2\
+netmask 255.255.255.0\
+gateway 192.168.2.1\
+        vlan-raw-device bond0\
+runcmd:\
+  - systemctl restart networking
+```
+
+
+
+### IP Address Management:
+
+* * * * *
+
+In Phase-1, the Cluster API Provider Packet (CAPP) will manage IP allotment to individual machines using Kubernetes ConfigMaps. This approach allows for tracking allocations and assigning available IP addresses dynamically.
+
+Example:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: capp-ip-allocations
+  namespace: cluster-api-provider-packet-system
+Data:
+  cidr: 192.168.2.0/24
+  allocations: |
+    {
+     "machine1": "192.168.2.2", 
+     "machine2": "192.168.2.3" 
+    }
+```
+
+In the example above, capp-ip-allocations ConfigMap in the cluster-api-provider-packet-system namespace tracks IP allocations. The cidr field specifies the IP range, while the allocations field is a JSON object mapping machine names to their allocated IP addresses.
+

From 894c6d940a71d56047dcdfe30bbeabfe6a9451af Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Wed, 21 Aug 2024 18:24:12 +0530
Subject: [PATCH 02/14] phase-1 docs for layer2 support in CAPP: version 2.0

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 443 +++++++++++++---------------------
 1 file changed, 164 insertions(+), 279 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index 33b34f71..36569edd 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -26,24 +26,23 @@ Background
 User stories
 ------------
 
-As a  cluster administrator
+As a 	**user of Cluster API provider Packet (CAPP)**
+I want 	**to configure L2 interfaces and define my own IP Address range**
+so that **machines are able to communicate over layer2 VLAN**
 
-- I want  to configure L2 interfaces and define my own IP Address range so that  machines are able to communicate over layer2 VLAN.
 
 * * * * *
 
-Goals (phase-1)
+Goals
 ===============
 
--   For the phase-1 of supporting Layer2 functionalities in CAPP, it will add support for [Hybrid Bonded](https://deploy.equinix.com/developers/docs/metal/layer2-networking/hybrid-bonded-mode/) and [Hybrid Unbonded](https://deploy.equinix.com/developers/docs/metal/layer2-networking/hybrid-unbonded-mode/) mode.
+In Phase 1 of integrating Layer2 support, the Cluster API Provider (CAPP) will focus on Bring Your Own (BYO) Infrastructure.
+Key objectives for this phase include:
+- Implementing Hybrid Bonded Mode and Hybrid Unbonded Modes to enhance Layer2 functionalities in CAPP.
+- Enabling CAPP to attach network ports to specific VLANs or VXLANs.
+- Allowing CAPP to configure Layer2 networking at the OS level on a metal node, including creating sub-interfaces and assigning IP addresses.
+- Ensuring CAPP can track the lifecycle of available IP addresses from VRF Range.
 
--   CAPP will be able to attach ports to specified VLAN.
-
--   CAPP will be able to create a new VLAN if required in the specified project and metro.
-
--   CAPP will be able to create VRF (Virtual Routing & Forwarding) and IP Reservations as specified in the spec.
-
--   CAPP will be able to configure networking at the OS level of a metal node.
 
 Non-Goals
 ===============
@@ -57,309 +56,195 @@ Proposal Design/Approach
 
 * * * * *
 
-### PacketCluster
-
-The idea is to introduce options to specify VLANs and VRFs configurations as a part of ClusterSpec.
-
-```go
-type  PacketClusterSpec struct {
-  ...
-
-// VLANConfig represents the configuration for layer2 VLAN.
- VLANConfig *VLANConfig            `json:"vlanConfig,omitempty"`
-
-// VRFConfig represents the configuration for Virtual  Router and Forwarding (VRF)
- VRFConfig *VRFConfig              `json:"vrfConfig,omitempty"`
-}
-```
-
-// VLANConfig represents the configuration for an existing or new VLAN
-```go
-type VLANConfig struct {
- // VLANID is the ID of the VLAN. It can be an existing VLAN or a new one to be created.
-  VLANID string `json:"vlanID"`
-
- // Description of the VLAN
- // +optional
-  Description string `json:"description,omitempty"`
-
- // CreateNew indicates whether a new VLAN needs to be created
- // +optional
-  CreateNew bool `json:"createNew,omitempty"`
-}
-```
-
--   For an existing VLAN
-```json
-{
-"vlanID": "existing-vlan-id",
-"description": "Existing VLAN",
-"createNew": false
-}
-```
-
--   For a new VLAN:
-
-```json
-{
-"vlanID": "new-vlan-id",
-"description": "New VLAN to be created",
-"createNew": true
-}
-```
-
-* * * * *
-
-
-// VRFConfig represents the configuration for an existing or new VRF
-
-```go
-type VRFConfig struct {
- // Name of the VRF. It can be an existing VRF or a new one to be created.
-  Name string `json:"name,omitempty"`
-
- // Description of the VRF
- // +optional
-  Description string `json:"description,omitempty"`
-
- // IP ranges that are allowed to access and communicate with this virtual router.
-  AllowedIPRanges []string `json:"allowedIPRanges,omitempty"`
-
- // CreateNew indicates whether a new VRF needs to be created\
- // +optional
-  CreateNew bool `json:"createNew,omitempty"`
-}
-```
-
-- For an existing VRF:
-
-```json
-{
-"name": "Existing VRF",
-"description": "Existing VRF description",
-"allowedIPRanges": ["192.168.1.0/21"],
-"createNew": false
-}
-```
-
-- For a new VRF:
-```json
-{
-"name": "New VRF",
-"description": "New VRF description",
-  "allowedIPRanges": ["10.0.0.0/24"],
-"createNew": true
-}
-```
-
-Users can specify different options for CAPP Controllers to create a new VLAN/VRF or to provide existing configuration details. Below is an example YAML configuration illustrating this:
-
-```yaml
-apiVersion: infrastructure.cluster.x-k8s.io/v1beta1\
-kind: PacketCluster
-metadata:
-  name: capi-quickstart
-  namespace: default
-spec:
-  metro: da
-  projectID:  7f713f3e-fe14-4386-a2fb-f22c29ba685e\
-  vipManager: CPEM
-  vlanConfig:
- "vlanID": "existing-vlan-id",
-"description": "Existing VLAN",
-"createNew": false
-  vrfConfig: 
-    "name": "New VRF",
-    "description": "New VRF description",
-    "allowedIPRanges": ["10.0.0.0/24"],
-    "createNew": true
-```
-
-In the above example, vlanConfig and vrfConfig are used to create a new VLAN and a new VRF with specific configurations, respectively.
-
-### Infrastructure Setup and Readiness
-
-* * * * *
-
-The PacketCluster Controller is responsible for setting up infrastructure. Once the setup is successfully completed, it marks itself as "Ready" and sets the condition NetworkInfrastructureReadyCondition to true. The controller will continue to follow this setup process and call Equinix Metal APIs to create or get VLANs and VRFs as needed before marking the infrastructure as Ready.
-
-This approach also helps avoid potential race conditions between the execution of user-data scripts and the application of port configurations at the API level. By ensuring the network configurations are established through the controller before any user-data scripts run, we can maintain a consistent and reliable infrastructure setup process.
-
-### APIs
-
-* * * * *
-
-1.  Create a new VLAN
-
-```
-curl -X POST
--H "Content-Type: application/json"
--H "X-Auth-Token: <API_TOKEN>"
-"https://api.equinix.com/metal/v1/projects/{id}/virtual-networks"
--d '{
- "vxlan": <integer>,
- "description": "<string>",
- "metro": "<string>"
- }'
-```
-
-2.  Create a new VRF
-```
-curl -X POST
--H "Content-Type: application/json"
--H "X-Auth-Token: <API_TOKEN>"
-"https://api.equinix.com/metal/v1/projects/{id}/vrfs"
--d '{
- "name": "<string>",
- "description": "<string>",
- "metro": "<metro_slug>",
- "ip_ranges": [
- "<cidr_address>"
-    ],
- "local_asn": <integer>
-}'
-```
-
-### PacketMachineTemplate
+**Understanding the context and problem space** : The problem space primarily revolves around the operating system (OS) and, to some extent, the cluster level. Specifically, it concerns how Cluster API (CAPP) clusters and machines are defined by IP addresses, networks, and gateways.
+A critical aspect of this space is how CAPP provisions infrastructure, particularly network infrastructure. This includes VLANs, gateways, virtual circuits, and IP address ranges such as elastic IPs or IP reservations. Additionally, it involves the management of VRFs and the attachment of these network resources to nodes, ensuring that newly created nodes have ports in a ready state for these attachments. The default approach will be Layer2 networking in a hybrid-bonded mode, though other configurations may also be supported in the future.
+This understanding forms the foundation for addressing the technical challenges in provisioning and managing network infrastructure with CAPP.
+ 
+**Bring Your Own Infrastructure (BYOI)**:
 
+The BYOI approach allows users to leverage their existing infrastructure, such as VLANs, VRFs, Metal Gateways, and similar components.
+In this model, users specify the IP ranges to be assigned to metal nodes on VLAN-tagged interfaces. Importantly, CAPP is not responsible for creating or managing this infrastructure, it is assumed to already exist.
+However, CAPP needs to be informed of the VLAN ID to attach the network port to the appropriate VLAN using the Equinix Metal (EM) API. This ensures that the network configuration aligns with the pre-existing infrastructure provided by the user.
 
-* * * * *
+### Custom Resource Changes:
+**PacketMachineTemplate**
 
-The idea is to introduce a new field Ports under spec which will contain different networking specifications of networking ports for a Equinix Metal Machine. This could look like the following:
+To support enhanced layer2 networking capabilities, we propose adding a new Ports field under the spec of the *PacketMachineTemplate*. This field will allow users to define various network port configurations for an Equinix Metal Machine. Below is an outline of the proposed changes:
 
 ```go
-// PacketMachineSpec defines the desired state of  PacketMachine.
-type  PacketMachineSpec struct {
-  ..
-// List of Port  Configurations on each Packet  Machine\
-  // +optional\
- Ports []Port `json:"ports"`
+// PacketMachineSpec defines the desired state of PacketMachine.
+ type PacketMachineSpec struct {
+   ..
+   // List of Port Configurations on each Packet Machine
+   // +optional
+   Ports []Port `json:"ports"`
 }
-```
 
-```go
 type Port struct {
- // name of the port e.g bond0
-    Name string `json:"name"`
- // port bonded or not - by default true
-    Bonded bool `json:"bonded,omitempty"`
- // convert port to layer 2. is false by default on new devices. changes result in /ports/id/convert/layer-[2|3] API calls\
-    Layer2 bool `json:"layer2"`
-
-    IPAddresses []IPAddress `json:"ip_addresses,omitempty"`
+     // name of the port e.g bond0,eth0 and eth1 for 2 NIC servers.
+     Name string `json:"name"`
+     // port bonded or not - by default true
+     Bonded bool `json:"bonded,omitempty"`
+     // convert port to layer 2. is false by default on new devices. changes result in /ports/id/convert/layer-[2|3] API calls
+     Layer2 bool `json:"layer2"`
+     // IPAddress configurations associated with this port
+     // These are typically IP Reservations carved out of VRF.
+     IPAddresses []IPAddress `json:"ip_addresses,omitempty"`	
 }
-// IPAddress represents an IP address configuration\
+// IPAddress represents an IP address configuration 
 type IPAddress struct {
-    Type string `json:"type"`
-
-    // VRF used to create the IP Address Reservation for this cluster\
-    VRFName string `json:"vrfName,omitempty"`
-
-    // IPAddressReservation to reserve for these cluster nodes.
-    IPAddressReservation string `json:"ipAddressReservation"`
-
-    // VLAN Id to join
-    VLANId string `json:"vlanId,omitempty"`
+    // IPAddressReservation to reserve for these cluster nodes.
+    // for eg: can be carved out of a VRF IP Range.
+    IPAddressReservation string `json:"ipAddressReservation"`	
+    // VLANs for EM API to find by vxlan, project, and metro match then attach to device. OS userdata template will also configure this VLAN on the bond device    
+    VXLANIDs []string `json:"vxlan_ids,omitempty"`
+    // UUID of VLANs to which this port should be assigned.
+    // Either VXLANID or VLANID should be provided.
+    VLANIDs []string  `vlan_ids,omitempty`
+    // IP Address of the gateway
+    Gateway string `gateway,omitempty`
 }
 ```
 
-Example:
+For example:
+The following example configures the bond0 port of each node in a cluster to a hybrid bonded mode, attaches vxlan_id with ID 1000 and assigns each node an IP address from range "192.168.2.0/24" with gateway 192.168.2.1
 
 ```yaml
-apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
-kind: PacketMachineTemplate\
+kind: PacketMachineTemplate
 metadata:
-    name: example-packet-machine-template
+  name: example-packet-machine-template
 spec:
-  template:
-    spec:
-        facility: ny5
-        metro: ny
-        plan: c3.small.x86
-        billingCycle: hourly
-        project: your-packet-project-id
-        sshKeys:
-        -  ssh-rsa  AAAAB3...your-public-key...
-        operatingSystem: ubuntu_20_04
-        ports: # network_ports in the API naming\
-        - bond0:
-            bonded: true  # default\
-            layer2: false
-            ip_addresses:
-            - type: vrf
-              vrfName: "vrf-01"
-              ipAddressReservation: "192.168.2.0/24"
-              vlanId: 1000
-        - eth2: # unbonded eth ports can use most of the same attributes bond ports can use\
-            layer2: true
+  template:
+    spec:
+      facility: ny5
+      metro: ny
+      plan: c3.small.x86
+      billingCycle: hourly
+      project: your-packet-project-id
+      sshKeys:
+        - ssh-rsa AAAAB3...your-public-key...
+      operatingSystem: ubuntu_20_04
+      ports:
+       -  name: bond0
+          layer2: false
+          ip_addresses:
+            - ipAddressReservation: "192.168.2.0/24"
+              vxlan_ids: [1000]
+              gateway: "192.168.2.1"
 ```
 
-The CAPP Operators while provisioning the metal node would:
-
-1.  Call the Equinix Metal API to attach Ports to the VLAN 
+The following example configures the eth1 port of each node in a cluster to a hybrid unbonded mode, removed the port from the bond, converts the port into a layer mode i.e attaches vxlan_id with ID 1001 and assigns each node an IP address from range "10.50.10.0/24" with gateway 10.50.10.1
 
-```
-curl -X POST
--H "Content-Type: application/json"
--H "X-Auth-Token: <API_TOKEN> "
-"https://api.equinix.com/metal/v1/ports/{id}/assign"
--d '{
-    "vnid": "<vlan_ID>"
-    }'
-```
-
-1.  Once all the ports are attached to the respective VLANs, CAPP needs to configure the networking on the server's operating system.
+```yaml
 
-2.  It will then GET the VRF and update it to reserve the IPAddressReservation.
+kind: PacketMachineTemplate
+metadata:
+  name: example-packet-machine-template
+spec:
+  template:
+    spec:
+      facility: ny5
+      metro: ny
+      plan: c3.small.x86
+      billingCycle: hourly
+      project: your-packet-project-id
+      sshKeys:
+        - ssh-rsa AAAAB3...your-public-key...
+      operatingSystem: ubuntu_20_04
+      ports:
+       - eth1:
+          bonded: false
+          layer2: true
+          ip_addresses:
+            - ipAddressReservation: "10.50.10.0/24"
+              vxlan_ids: [1001]
+              gateway: "10.50.10.1"
 
-```
-curl -X POST
--H 'Content-Type: application/json'
--H "X-Auth-Token: <API_TOKEN>"
-"https://api.equinix.com/metal/v1/projects/{id}/ips"
--d '{
-    "cidr": <integer>,
-    "network": "<ip_address>",
-    "type": "vrf",
-    "vrf_id": "<UUID>"
-}'
 ```
 
-1.  For each Machine, CAPP needs to pick an IP Address from the above range and assign it at the OS Level. In Phase 1, it will be done by CAPP itself (explained below) and later in phase-2, an IPAM Provider will be implemented as per CAPI standards.
+### APIs:
 
-2.  Once it receives an IP Address that is non-conflicting, It will create subinterfaces to handle the tagged traffic over the VLAN and assign the IP Address on that sub-interface. Hybrid Bonded mode does not support untagged VLAN traffic or setting a Native VLAN.
-
-3.  In order to configure networking on the OS, user-data and cloud-config format could be utilized as shown below in the example.
+* * * * *
 
-Example:
+Following are some of the APIs provided by EM, that would be used:
+1. **Convert the port to a layer2 port**:
+
+    a. https://deploy.equinix.com/developers/api/metal/#tag/Ports/operation/convertLayer2
+    b. Endpoint: https://api.equinix.com/metal/v1/ports/{id}/convert/layer-2
+    c. Requied Params : vnid (VLAN ID) 
+
+2. **Assign a port to a virtual network (VLAN)**:
+
+    a. https://deploy.equinix.com/developers/api/metal/#tag/Ports/operation/assignPort
+
+    b. Endpoint: https://api.equinix.com/metal/v1/ports/{id}/assign
+Requied Params : vnid (VLAN ID)
+	c. Type: POST
+    d. Batch Mode
+    ```
+    curl -X POST \
+    -H "Content-Type: application/json" \
+    -H "X-Auth-Token: <API_TOKEN> " \
+    "https://api.equinix.com/metal/v1/ports/{id}/vlan-assignments/batches" \
+    -d '{
+        "vlan_assignments": [
+            {
+                "vlan": "string",
+                "state": "assigned"
+            },
+            {
+                "vlan": "string",
+                "state": "assigned"
+            },
+        ]
+    }'
+    ```
+
+3. **Device Events API**:
+ 	a. Endpoint:  `https://api.equinix.com/metal/v1/devices/<id>/events`
+
+4. **Remove port from the bond**
+    a. Endpoint: 
+    ```
+    curl -X POST \
+        -H "Content-Type: application/json" \
+        -H "X-Auth-Token: <API_TOKEN>" \
+        "https://api.equinix.com/metal/v1/ports/{id}/disbond" \
+        -d '{
+            "bulk_disable": false
+        }'
+    ```
+
+
+### User-Data Script for Network Configuration
+To configure the operating system (OS), create new sub-interfaces for handling VLAN-tagged traffic, and assign IP addresses to those sub-interfaces, a user-data script is required to run at the time of OS boot.
+Below is the user-data script that would be used (WIP)
 
 ```sh
-#cloud-config\
-write_files:\
-  - path: /etc/network/interfaces
-
-    append: true\
-    content: |
-
-      auto bond0.${VLAN_ID_0}\
-        iface bond0.${VLAN_ID_0} inet static
-
-        pre-up sleep 5\
-address 192.168.2.2\
-netmask 255.255.255.0\
-gateway 192.168.2.1\
-        vlan-raw-device bond0\
-runcmd:\
-  - systemctl restart networking
-```
 
+```
+### Layer 2 Networking Setup by the CAPP Operator
+When provisioning a metal node with Layer 2 networking, the Cluster API Provider (CAPP) Operator will perform the following steps:
+1. **Create a ConfigMap for IP Address Management**: The operator will create a new ConfigMap named <cluster_name-port_name> for each port to manage IP addresses. This ConfigMap is critical for tracking and allocating IP addresses as detailed in the *IP Address Management* section.
+2. **Select an Available IP Address**: CAPP will select an available IP address from the ConfigMap to be assigned to the machine, node, or server being provisioned.
+3. **Generate User-Data Script**: Using Go templates, CAPP will substitute the necessary variables in the user-data script, such as port name, IP address, gateway, and VXLAN. These values are provided by the user through the custom resource definition.
+4. **Submit Device Creation Request**: CAPP will then submit a request to create the device, incorporating the generated user-data script for OS and network configuration.
+5. **Verify Network Configuration**: After the machine or device is successfully provisioned, CAPP will poll the device events API to check whether the network configuration was successful. If not, it will handle the failure or timeout as needed.
+6. **Perform Post-Provisioning Network Operations**: Once the device is provisioned and the network configuration from the user-data script is in place, CAPP will make calls to the /ports API to perform additional operations. These include assigning the VLAN to the port, converting the port to Layer 2 if required, and other necessary configurations.
+
+### Explanation of send_user_state_event Function
+The send_user_state_event function in the script is responsible for sending status updates to the user_state_url fetched from Equinix Metadata API. The Metadata API is a service available on every Equinix Metal server instance that allows the server to access and share various data about itself. Here’s how the function works:
+1. **Retrieve the user_state_url**: The script fetches the user_state_url from the Equinix Metadata API. This URL is used to send custom user state events that report on the progress or status of the server's configuration.
+2. **Prepare the Event Data**: The function constructs a JSON payload containing the state, code, and message. The jq tool is used to create this JSON object dynamically, based on the input parameters.
+3. **Send the Event**: The constructed JSON data is then sent to the user_state_url via a POST request. This allows the system to log the state of the network configuration process (e.g., "running," "succeeded," or "failed") along with an appropriate status code and message.
+This approach enables tracking of the server's state during the boot process, particularly for critical operations like network configuration.
 
 
 ### IP Address Management:
 
 * * * * *
 
-In Phase-1, the Cluster API Provider Packet (CAPP) will manage IP allotment to individual machines using Kubernetes ConfigMaps. This approach allows for tracking allocations and assigning available IP addresses dynamically.
+In Phase-1, the Cluster API Provider Packet (CAPP) will manage IP allotment to individual machines using Kubernetes Configmaps. This approach allows for tracking allocations and assigning available IP addresses dynamically.
 
 Example:
 

From 75f46a7874516f651783bee7dd80173f2845bed5 Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Thu, 22 Aug 2024 14:20:18 +0530
Subject: [PATCH 03/14] add user-data script

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 104 ++++++++++++++++++++++++++++++++++
 1 file changed, 104 insertions(+)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index 36569edd..63d0b74c 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -221,6 +221,110 @@ To configure the operating system (OS), create new sub-interfaces for handling V
 Below is the user-data script that would be used (WIP)
 
 ```sh
+#cloud-config
+
+# Write the provided script to a temporary file
+write_files:
+  - path: /tmp/final_configuration.sh
+    permissions: '0755'
+    content: |
+      #!/bin/bash
+      set -euo pipefail
+
+      echo "Running final configuration commands"
+      apt-get update -qq
+      apt-get install -y -qq jq vlan
+
+      # Generate the network configuration and append it to /etc/network/interfaces for each VLAN-tagged sub-interface.
+      {
+        echo "{{ range $index, $vlan := .VLANs }}"
+        echo "auto {{ $vlan.PortName }}.{{ $vlan.ID }}"
+        echo "iface {{ $vlan.PortName }}.{{ $vlan.ID }} inet static"
+        echo " pre-up sleep 5"
+        echo " address {{ $vlan.IPAddress }}"
+        echo " netmask {{ $vlan.Netmask }}"
+        echo " gateway {{ $vlan.Gateway }}"
+        echo " vlan-raw-device {{ $vlan.PortName }}"
+        echo "{{ end }}"
+      } >> /etc/network/interfaces
+
+      echo "VLAN configuration appended to /etc/network/interfaces."
+
+      # Function to send user state events
+      url="$(curl -sf https://metadata.platformequinix.com/metadata | jq -r .user_state_url)"
+      send_user_state_event() {
+        local state="$1"
+        local code="$2"
+        local message="$3"
+        local data
+
+        data=$(jq -n --arg state "$state" --arg code "$code" --arg message "$message" \
+               '{state: $state, code: ($code | tonumber), message: $message}')
+        
+        curl -s -X POST -d "$data" "$url" || echo "Failed to send user state event"
+      }
+
+      send_user_state_event running 1000 "Configuring Network"
+
+      # Restart networking and check for failures
+      if ! systemctl restart networking; then
+        echo "Network restart failed" >&2
+        send_user_state_event failed 1002 "Network restart failed"
+        exit 1
+      fi
+
+      # Verify network configuration
+      verification_failed=false
+      {{ range $index, $vlan := .VLANs }}
+      if ip addr show {{ $vlan.PortName }}.{{ $vlan.ID }} | grep -q {{ $vlan.IPAddress }}; then
+        echo "Configuration for VLAN {{ $vlan.ID }} on {{ $vlan.PortName }} with IP {{ $vlan.IPAddress }} successful"
+      else
+        echo "Configuration for VLAN {{ $vlan.ID }} on {{ $vlan.PortName }} with IP {{ $vlan.IPAddress }} failed" >&2
+        verification_failed=true
+      fi
+      {{ end }}
+
+      if [ "$verification_failed" = true ]; then
+        send_user_state_event failed 1002 "Network configuration failed"
+        exit 1
+      else
+        send_user_state_event succeeded 1001 "Network configuration successful"
+      fi
+
+# Commands to run after the script is written
+runcmd:
+  - |
+      # Fetch metadata and set up network interfaces
+      metadata=$(curl -sf https://metadata.platformequinix.com/metadata)
+
+      # Extract MAC addresses for eth0 and eth1
+      mac_eth0=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth0") | .mac')
+      mac_eth1=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth1") | .mac')
+
+      # Function to find interface name by MAC address
+      find_interface_by_mac() {
+        local mac="$1"
+        for iface in $(ls /sys/class/net/); do
+          iface_mac=$(ethtool -P "$iface" 2>/dev/null | awk '{print $NF}')
+          if [ "$iface_mac" == "$mac" ]; then
+            echo "$iface"
+            return
+          fi
+        done
+        echo "Interface not found for MAC $mac" >&2
+        return 1
+      }
+
+      # Find interface names for eth0 and eth1
+      iface_eth0=$(find_interface_by_mac "$mac_eth0")
+      iface_eth1=$(find_interface_by_mac "$mac_eth1")
+
+      # Replace eth0 and eth1 in the script with the actual interface names
+      sed -i "s/eth0/${iface_eth0}/g" /tmp/final_configuration.sh
+      sed -i "s/eth1/${iface_eth1}/g" /tmp/final_configuration.sh
+
+      # Execute the modified script
+      bash /tmp/final_configuration.sh
 
 ```
 ### Layer 2 Networking Setup by the CAPP Operator

From 86c1af428e39d3fb75d8712a208d9cb77d2e1646 Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Thu, 22 Aug 2024 14:21:33 +0530
Subject: [PATCH 04/14] add user-data script

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index 63d0b74c..862dd54d 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -218,7 +218,7 @@ Requied Params : vnid (VLAN ID)
 
 ### User-Data Script for Network Configuration
 To configure the operating system (OS), create new sub-interfaces for handling VLAN-tagged traffic, and assign IP addresses to those sub-interfaces, a user-data script is required to run at the time of OS boot.
-Below is the user-data script that would be used (WIP)
+Below is the user-data script that would be used.
 
 ```sh
 #cloud-config
@@ -327,6 +327,9 @@ runcmd:
       bash /tmp/final_configuration.sh
 
 ```
+
+The CAPP will use go-templates to substitute the placeholders with appropriate values given by the user.
+
 ### Layer 2 Networking Setup by the CAPP Operator
 When provisioning a metal node with Layer 2 networking, the Cluster API Provider (CAPP) Operator will perform the following steps:
 1. **Create a ConfigMap for IP Address Management**: The operator will create a new ConfigMap named <cluster_name-port_name> for each port to manage IP addresses. This ConfigMap is critical for tracking and allocating IP addresses as detailed in the *IP Address Management* section.

From 4942c7198302ceb1964ca0a742b4aa7221fd5bde Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Fri, 23 Aug 2024 12:01:44 +0530
Subject: [PATCH 05/14] fixes to user-data script

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 107 +++++++++++++++++-----------------
 1 file changed, 52 insertions(+), 55 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index 862dd54d..e9877d36 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -222,8 +222,12 @@ Below is the user-data script that would be used.
 
 ```sh
 #cloud-config
+package_update: true
+package_upgrade: true
+packages:
+  - jq
+  - vlan
 
-# Write the provided script to a temporary file
 write_files:
   - path: /tmp/final_configuration.sh
     permissions: '0755'
@@ -236,17 +240,17 @@ write_files:
       apt-get install -y -qq jq vlan
 
       # Generate the network configuration and append it to /etc/network/interfaces for each VLAN-tagged sub-interface.
-      {
-        echo "{{ range $index, $vlan := .VLANs }}"
-        echo "auto {{ $vlan.PortName }}.{{ $vlan.ID }}"
-        echo "iface {{ $vlan.PortName }}.{{ $vlan.ID }} inet static"
-        echo " pre-up sleep 5"
-        echo " address {{ $vlan.IPAddress }}"
-        echo " netmask {{ $vlan.Netmask }}"
-        echo " gateway {{ $vlan.Gateway }}"
-        echo " vlan-raw-device {{ $vlan.PortName }}"
-        echo "{{ end }}"
-      } >> /etc/network/interfaces
+      cat <<EOL >> /etc/network/interfaces
+{{- range .VLANs }}
+auto {{ .PortName }}.{{ .ID }}
+iface {{ .PortName }}.{{ .ID }} inet static
+  pre-up sleep 5
+  address {{ .IPAddress }}
+  netmask {{ .Netmask }}
+  gateway {{ .Gateway }}
+  vlan-raw-device {{ .PortName }}
+{{- end }}
+EOL
 
       echo "VLAN configuration appended to /etc/network/interfaces."
 
@@ -260,29 +264,24 @@ write_files:
 
         data=$(jq -n --arg state "$state" --arg code "$code" --arg message "$message" \
                '{state: $state, code: ($code | tonumber), message: $message}')
-        
+
         curl -s -X POST -d "$data" "$url" || echo "Failed to send user state event"
       }
 
       send_user_state_event running 1000 "Configuring Network"
 
-      # Restart networking and check for failures
-      if ! systemctl restart networking; then
-        echo "Network restart failed" >&2
-        send_user_state_event failed 1002 "Network restart failed"
-        exit 1
-      fi
+      systemctl restart networking
 
       # Verify network configuration
       verification_failed=false
-      {{ range $index, $vlan := .VLANs }}
-      if ip addr show {{ $vlan.PortName }}.{{ $vlan.ID }} | grep -q {{ $vlan.IPAddress }}; then
-        echo "Configuration for VLAN {{ $vlan.ID }} on {{ $vlan.PortName }} with IP {{ $vlan.IPAddress }} successful"
+{{- range .VLANs }}
+      if ip addr show {{ .PortName }}.{{ .ID }} | grep -q {{ .IPAddress }}; then
+        echo "Configuration for VLAN {{ .ID }} on {{ .PortName }} with IP {{ .IPAddress }} successful"
       else
-        echo "Configuration for VLAN {{ $vlan.ID }} on {{ $vlan.PortName }} with IP {{ $vlan.IPAddress }} failed" >&2
+        echo "Configuration for VLAN {{ .ID }} on {{ .PortName }} with IP {{ .IPAddress }} failed" >&2
         verification_failed=true
       fi
-      {{ end }}
+{{- end }}
 
       if [ "$verification_failed" = true ]; then
         send_user_state_event failed 1002 "Network configuration failed"
@@ -291,41 +290,39 @@ write_files:
         send_user_state_event succeeded 1001 "Network configuration successful"
       fi
 
-# Commands to run after the script is written
 runcmd:
-  - |
-      # Fetch metadata and set up network interfaces
-      metadata=$(curl -sf https://metadata.platformequinix.com/metadata)
-
-      # Extract MAC addresses for eth0 and eth1
-      mac_eth0=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth0") | .mac')
-      mac_eth1=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth1") | .mac')
-
-      # Function to find interface name by MAC address
-      find_interface_by_mac() {
-        local mac="$1"
-        for iface in $(ls /sys/class/net/); do
-          iface_mac=$(ethtool -P "$iface" 2>/dev/null | awk '{print $NF}')
-          if [ "$iface_mac" == "$mac" ]; then
-            echo "$iface"
-            return
-          fi
-        done
-        echo "Interface not found for MAC $mac" >&2
-        return 1
-      }
-
-      # Find interface names for eth0 and eth1
-      iface_eth0=$(find_interface_by_mac "$mac_eth0")
-      iface_eth1=$(find_interface_by_mac "$mac_eth1")
+- |
+    # Fetch metadata and set up network interfaces
+    metadata=$(curl -sf https://metadata.platformequinix.com/metadata)
+
+    # Extract MAC addresses for eth0 and eth1
+    mac_eth0=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth0") | .mac')
+    mac_eth1=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth1") | .mac')
+
+    # Function to find interface name by MAC address
+    find_interface_by_mac() {
+      local mac="$1"
+      for iface in $(ls /sys/class/net/); do
+        iface_mac=$(ethtool -P "$iface" 2>/dev/null | awk '{print $NF}')
+        if [ "$iface_mac" == "$mac" ]; then
+          echo "$iface"
+          return
+        fi
+      done
+      echo "Interface not found for MAC $mac" >&2
+      return 1
+    }
 
-      # Replace eth0 and eth1 in the script with the actual interface names
-      sed -i "s/eth0/${iface_eth0}/g" /tmp/final_configuration.sh
-      sed -i "s/eth1/${iface_eth1}/g" /tmp/final_configuration.sh
+    # Find interface names for eth0 and eth1
+    iface_eth0=$(find_interface_by_mac "$mac_eth0")
+    iface_eth1=$(find_interface_by_mac "$mac_eth1")
 
-      # Execute the modified script
-      bash /tmp/final_configuration.sh
+    # Replace eth0 and eth1 in the script with the actual interface names
+    sed -i "s/eth0/${iface_eth0}/g" /tmp/final_configuration.sh
+    sed -i "s/eth1/${iface_eth1}/g" /tmp/final_configuration.sh
 
+    # Execute the modified script
+    bash /tmp/final_configuration.sh
 ```
 
 The CAPP will use go-templates to substitute the placeholders with appropriate values given by the user.

From bb1401eddee1341f0e476a9ecaec60422f34b00a Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Tue, 27 Aug 2024 12:14:49 +0530
Subject: [PATCH 06/14] changes to user data scripts

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 122 +++++++++++++++++++++-------------
 1 file changed, 76 insertions(+), 46 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index e9877d36..b6bc63cb 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -229,7 +229,7 @@ packages:
   - vlan
 
 write_files:
-  - path: /tmp/final_configuration.sh
+  - path: /var/lib/capi_network_settings/final_configuration.sh
     permissions: '0755'
     content: |
       #!/bin/bash
@@ -239,18 +239,23 @@ write_files:
       apt-get update -qq
       apt-get install -y -qq jq vlan
 
+      modprobe 8021q
+      echo "8021q" >> /etc/modules
+
       # Generate the network configuration and append it to /etc/network/interfaces for each VLAN-tagged sub-interface.
       cat <<EOL >> /etc/network/interfaces
-{{- range .VLANs }}
-auto {{ .PortName }}.{{ .ID }}
-iface {{ .PortName }}.{{ .ID }} inet static
-  pre-up sleep 5
-  address {{ .IPAddress }}
-  netmask {{ .Netmask }}
-  gateway {{ .Gateway }}
-  vlan-raw-device {{ .PortName }}
-{{- end }}
-EOL
+{{ range .VLANs }}
+      auto {{ .PortName }}.{{ .ID }}
+      iface {{ .PortName }}.{{ .ID }} inet static
+        pre-up sleep 5
+        address {{ .IPAddress }}
+        netmask {{ .Netmask }}
+      {{- if .Gateway }}
+        gateway {{ .Gateway }}
+      {{- end }}
+        vlan-raw-device {{ .PortName }}
+{{ end }}
+      EOL
 
       echo "VLAN configuration appended to /etc/network/interfaces."
 
@@ -271,17 +276,16 @@ EOL
       send_user_state_event running 1000 "Configuring Network"
 
       systemctl restart networking
-
       # Verify network configuration
       verification_failed=false
-{{- range .VLANs }}
+{{ range .VLANs }}
       if ip addr show {{ .PortName }}.{{ .ID }} | grep -q {{ .IPAddress }}; then
         echo "Configuration for VLAN {{ .ID }} on {{ .PortName }} with IP {{ .IPAddress }} successful"
       else
         echo "Configuration for VLAN {{ .ID }} on {{ .PortName }} with IP {{ .IPAddress }} failed" >&2
         verification_failed=true
       fi
-{{- end }}
+{{ end }}
 
       if [ "$verification_failed" = true ]; then
         send_user_state_event failed 1002 "Network configuration failed"
@@ -289,40 +293,66 @@ EOL
       else
         send_user_state_event succeeded 1001 "Network configuration successful"
       fi
+    
+  - path: /var/lib/capi_network_settings/initial_configuration.sh
+    permissions: '0755'
+    content: |
+      #!/bin/bash
+      set -eu
+      
+      # Fetch metadata from Equinix Metal
+      metadata=$(curl https://metadata.platformequinix.com/metadata)
+      
+      # Extract MAC addresses for eth0 and eth1
+      mac_eth0=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth0") | .mac')
+      mac_eth1=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth1") | .mac')
+      
+      # Check if MAC addresses were successfully extracted
+      if [ -z "$mac_eth0" ] || [ -z "$mac_eth1" ]; then
+        echo "Error: Failed to extract MAC addresses" >&2
+        exit 1
+      fi
+      
+      # Display extracted MAC addresses
+      echo "Extracted MAC addresses - eth0: $mac_eth0, eth1: $mac_eth1"
+      
+      # Function to find network interface by MAC address
+      find_interface_by_mac() {
+        local mac="$1"
+        for iface in $(ls /sys/class/net/); do
+          iface_mac=$(ethtool -P "$iface" 2>/dev/null | awk '{print $NF}')
+          if [ "$iface_mac" == "$mac" ]; then
+            echo "$iface"
+            return
+          fi
+        done
+        echo "Interface not found for MAC $mac" >&2
+        return 1
+      }
+      
+      # Find interfaces for eth0 and eth1 MAC addresses
+      iface_eth0=$(find_interface_by_mac "$mac_eth0")
+      iface_eth1=$(find_interface_by_mac "$mac_eth1")
+      
+      # Check and replace eth0 in /tmp/final_configuration.sh
+      if grep -q "eth0" /tmp/final_configuration.sh; then
+        sed -i "s/eth0/${iface_eth0}/g" /tmp/final_configuration.sh
+        echo "Replaced eth0 with ${iface_eth0} in /tmp/final_configuration.sh"
+      else
+        echo "No occurrences of eth0 found in /tmp/final_configuration.sh. No changes made."
+      fi
+      
+      # Check and replace eth1 in /tmp/final_configuration.sh
+      if grep -q "eth1" /tmp/final_configuration.sh; then
+        sed -i "s/eth1/${iface_eth1}/g" /tmp/final_configuration.sh
+        echo "Replaced eth1 with ${iface_eth1} in /tmp/final_configuration.sh"
+      else
+        echo "No occurrences of eth1 found in /tmp/final_configuration.sh. No changes made."
+      fi
 
 runcmd:
-- |
-    # Fetch metadata and set up network interfaces
-    metadata=$(curl -sf https://metadata.platformequinix.com/metadata)
-
-    # Extract MAC addresses for eth0 and eth1
-    mac_eth0=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth0") | .mac')
-    mac_eth1=$(echo "$metadata" | jq -r '.network.interfaces[] | select(.name == "eth1") | .mac')
-
-    # Function to find interface name by MAC address
-    find_interface_by_mac() {
-      local mac="$1"
-      for iface in $(ls /sys/class/net/); do
-        iface_mac=$(ethtool -P "$iface" 2>/dev/null | awk '{print $NF}')
-        if [ "$iface_mac" == "$mac" ]; then
-          echo "$iface"
-          return
-        fi
-      done
-      echo "Interface not found for MAC $mac" >&2
-      return 1
-    }
-
-    # Find interface names for eth0 and eth1
-    iface_eth0=$(find_interface_by_mac "$mac_eth0")
-    iface_eth1=$(find_interface_by_mac "$mac_eth1")
-
-    # Replace eth0 and eth1 in the script with the actual interface names
-    sed -i "s/eth0/${iface_eth0}/g" /tmp/final_configuration.sh
-    sed -i "s/eth1/${iface_eth1}/g" /tmp/final_configuration.sh
-
-    # Execute the modified script
-    bash /tmp/final_configuration.sh
+  - /var/lib/capi_network_settings/initial_configuration.sh
+  - /var/lib/capi_network_settings/final_configuration.sh
 ```
 
 The CAPP will use go-templates to substitute the placeholders with appropriate values given by the user.

From c32966fea44781f46ea0839b7108855cd2cc1441 Mon Sep 17 00:00:00 2001
From: Rahul Sawra <r.sawra@gmail.com>
Date: Tue, 27 Aug 2024 13:09:53 +0530
Subject: [PATCH 07/14] Apply suggestions from code review

Co-authored-by: Marques Johansson <mjohansson@equinix.com>
Co-authored-by: Nirav Parikh <52062717+niravparikh05@users.noreply.github.com>
---
 docs/layer2/layer2_support.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index b6bc63cb..23505585 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -40,8 +40,8 @@ In Phase 1 of integrating Layer2 support, the Cluster API Provider (CAPP) will f
 Key objectives for this phase include:
 - Implementing Hybrid Bonded Mode and Hybrid Unbonded Modes to enhance Layer2 functionalities in CAPP.
 - Enabling CAPP to attach network ports to specific VLANs or VXLANs.
-- Allowing CAPP to configure Layer2 networking at the OS level on a metal node, including creating sub-interfaces and assigning IP addresses.
-- Ensuring CAPP can track the lifecycle of available IP addresses from VRF Range.
+- Allowing CAPP to configure Layer2 networking at the OS level on a metal node, including creating sub-interfaces, assigning IP addresses, and assigning VLANs.
+- Ensuring CAPP can track the lifecycle of available IP addresses from a specified range, which may be arbitrarily defined or sourced from VRF
 
 
 Non-Goals
@@ -152,7 +152,7 @@ spec:
         - ssh-rsa AAAAB3...your-public-key...
       operatingSystem: ubuntu_20_04
       ports:
-       - eth1:
+       - name: eth1
           bonded: false
           layer2: true
           ip_addresses:

From eaf5412def34f18bf2bf109604e0c3f8aa895584 Mon Sep 17 00:00:00 2001
From: Rahul Sawra <r.sawra@gmail.com>
Date: Tue, 27 Aug 2024 13:14:31 +0530
Subject: [PATCH 08/14] Apply suggestions from code review

Co-authored-by: Marques Johansson <mjohansson@equinix.com>
---
 docs/layer2/layer2_support.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index 23505585..1159c700 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -367,7 +367,7 @@ When provisioning a metal node with Layer 2 networking, the Cluster API Provider
 6. **Perform Post-Provisioning Network Operations**: Once the device is provisioned and the network configuration from the user-data script is in place, CAPP will make calls to the /ports API to perform additional operations. These include assigning the VLAN to the port, converting the port to Layer 2 if required, and other necessary configurations.
 
 ### Explanation of send_user_state_event Function
-The send_user_state_event function in the script is responsible for sending status updates to the user_state_url fetched from Equinix Metadata API. The Metadata API is a service available on every Equinix Metal server instance that allows the server to access and share various data about itself. Here’s how the function works:
+The send_user_state_event function in the script is responsible for sending status updates to the user_state_url fetched from Equinix Metadata API. The Metadata API is a service available on every Equinix Metal server instance (while in Layer3 or Hybrid mode) that allows the server to access and share various data about itself. Here’s how the function works:
 1. **Retrieve the user_state_url**: The script fetches the user_state_url from the Equinix Metadata API. This URL is used to send custom user state events that report on the progress or status of the server's configuration.
 2. **Prepare the Event Data**: The function constructs a JSON payload containing the state, code, and message. The jq tool is used to create this JSON object dynamically, based on the input parameters.
 3. **Send the Event**: The constructed JSON data is then sent to the user_state_url via a POST request. This allows the system to log the state of the network configuration process (e.g., "running," "succeeded," or "failed") along with an appropriate status code and message.

From bdaa1c75b6fde1c7cb422773782e4ce451f3de87 Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Thu, 29 Aug 2024 14:08:03 +0530
Subject: [PATCH 09/14] minor fix

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index 1159c700..bf68b5b0 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -93,9 +93,9 @@ type Port struct {
 }
 // IPAddress represents an IP address configuration 
 type IPAddress struct {
-    // IPAddressReservation to reserve for these cluster nodes.
+    // Addresses to reserve for these ports.
     // for eg: can be carved out of a VRF IP Range.
-    IPAddressReservation string `json:"ipAddressReservation"`	
+    Address string `json:"address"`
     // VLANs for EM API to find by vxlan, project, and metro match then attach to device. OS userdata template will also configure this VLAN on the bond device    
     VXLANIDs []string `json:"vxlan_ids,omitempty"`
     // UUID of VLANs to which this port should be assigned.

From ecf951d9bab745cf19027a56e9ef50970d9ae594 Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Tue, 3 Sep 2024 18:43:39 +0530
Subject: [PATCH 10/14] changes to design doc

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 153 ++++++++++++++++++++++++----------
 1 file changed, 109 insertions(+), 44 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index bf68b5b0..155baaa0 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -67,6 +67,75 @@ In this model, users specify the IP ranges to be assigned to metal nodes on VLAN
 However, CAPP needs to be informed of the VLAN ID to attach the network port to the appropriate VLAN using the Equinix Metal (EM) API. This ensures that the network configuration aligns with the pre-existing infrastructure provided by the user.
 
 ### Custom Resource Changes:
+
+PacketCluster and PacketMachineTemplate will be extended to support the new network configurations for Layer2 networking.
+PacketCluster will have a new field called Networks, which will be a list of NetworkSpec objects. NetworkSpec objects define different networks that can be attached to the PacketCluster. Each NetworkSpec object will have the following fields:
+- Name: Name of the network, e.g., "storage VLAN" (optional)
+- Description: Description of the network, e.g., "Storage network for VMs" (optional)
+- IPAddresses: IP address range for the cluster network, e.g, Virtual Routing and Forwarding (VRF) . This field will be a list of strings, where each string represents an IP address range.
+- Assignment: Component responsible for allocating IP addresses to the machines, either cluster-api or dhcp.
+- Gateway: Default gateway for the network (optional)
+
+**PacketCluster**
+```go
+type PacketClusterSpec struct {
+  ...
+    // Networks is a list of network configurations for the PacketCluster
+    Networks []NetworkSpec `json:"networks,omitempty"`
+}
+
+// +kubebuilder:validation:Enum=cluster-api,dhcp
+type AssignmentType string
+
+const (
+    AssignmentClusterAPI AssignmentType = "cluster-api"
+    AssignmentDHCP       AssignmentType = "dhcp"
+)
+
+// NetworkSpec defines the network configuration for a PacketCluster
+type NetworkSpec struct {
+    // Name of the network, e.g. "storage VLAN", is optional
+    // +optional
+    Name        string         `json:"name,omitempty"`
+    // Description of the network, e.g. "Storage network", is optional
+    // +optional
+    Description string         `json:"description,omitempty"`
+    // IpAddressRange for the cluster network for eg: VRF IP Ranges
+    IPAddresses []string      `json:"ipAddresses,omitempty"`
+    // Assignment is component responsible for allocating IP addresses to the machines, either cluster-api or dhcp
+    Assignment  AssignmentType `json:"assignment,omitempty"`
+    // Default gateway for the network
+    // +optional
+    Gateway     string        `json:"gateway,omitempty"`
+}
+```
+
+The following example configures a network named "storage VLAN" with VXLAN ID 1000, IP address range 192.168.10.0/24, and a default gateway of
+192.168.10.1. The IP addresses are assigned by the cluster-api component.
+
+```yaml
+kind: PacketCluster
+metadata:
+  name: example-packet-cluster
+spec:
+  template:
+    spec:
+      facility: ny5
+      metro: ny
+      plan: c3.small.x86
+      billingCycle: hourly
+      project: your-packet-project-id
+      sshKeys:
+        - ssh-rsa AAAAB3...your-public-key...
+      operatingSystem: ubuntu_20_04
+      networks:
+        - name: Storage VLAN
+          description: Storage network for VMs
+          ipAddresses: ["192.168.0.0/16"]
+          assignment: cluster-api
+          gateway: "192.168.10.1"
+```
+
 **PacketMachineTemplate**
 
 To support enhanced layer2 networking capabilities, we propose adding a new Ports field under the spec of the *PacketMachineTemplate*. This field will allow users to define various network port configurations for an Equinix Metal Machine. Below is an outline of the proposed changes:
@@ -78,6 +147,9 @@ To support enhanced layer2 networking capabilities, we propose adding a new Port
    // List of Port Configurations on each Packet Machine
    // +optional
    Ports []Port `json:"ports"`
+   // List of Routes to be configured on the Packet Machine
+    // +optional
+   Routes      []RouteSpec     `json:"routes,omitempty"`
 }
 
 type Port struct {
@@ -89,7 +161,7 @@ type Port struct {
      Layer2 bool `json:"layer2"`
      // IPAddress configurations associated with this port
      // These are typically IP Reservations carved out of VRF.
-     IPAddresses []IPAddress `json:"ip_addresses,omitempty"`	
+     IPAddresses []IPAddress `json:"ip_addresses,omitempty"`
 }
 // IPAddress represents an IP address configuration 
 type IPAddress struct {
@@ -97,17 +169,23 @@ type IPAddress struct {
     // for eg: can be carved out of a VRF IP Range.
     Address string `json:"address"`
     // VLANs for EM API to find by vxlan, project, and metro match then attach to device. OS userdata template will also configure this VLAN on the bond device    
-    VXLANIDs []string `json:"vxlan_ids,omitempty"`
-    // UUID of VLANs to which this port should be assigned.
-    // Either VXLANID or VLANID should be provided.
-    VLANIDs []string  `vlan_ids,omitempty`
+    VXLANID int `json:"vxlanId,omitempty"`
     // IP Address of the gateway
     Gateway string `gateway,omitempty`
+    // Subnet Size for per machine
+    // +optional
+    SubnetSize string `json:"subnetSize,omitempty"`
+}
+
+// RouteSpec defines the static route configuration for a PacketMachine.
+type RouteSpec struct {
+    Destination string `json:"destination"`
+    Gateway     string `json:"gateway"`
 }
 ```
 
 For example:
-The following example configures the bond0 port of each node in a cluster to a hybrid bonded mode, attaches vxlan_id with ID 1000 and assigns each node an IP address from range "192.168.2.0/24" with gateway 192.168.2.1
+The following example configures the bond0 port of each node in a cluster to a hybrid bonded mode, attaches vxlanId with ID 1000 and assigns each node an IP address from range "192.168.10.0/24" with gateway 192.168.10.1
 
 ```yaml
 kind: PacketMachineTemplate
@@ -125,41 +203,20 @@ spec:
         - ssh-rsa AAAAB3...your-public-key...
       operatingSystem: ubuntu_20_04
       ports:
-       -  name: bond0
+        - name: bond0
           layer2: false
           ip_addresses:
-            - ipAddressReservation: "192.168.2.0/24"
-              vxlan_ids: [1000]
-              gateway: "192.168.2.1"
-```
-
-The following example configures the eth1 port of each node in a cluster to a hybrid unbonded mode, removed the port from the bond, converts the port into a layer mode i.e attaches vxlan_id with ID 1001 and assigns each node an IP address from range "10.50.10.0/24" with gateway 10.50.10.1
-
-```yaml
-
-kind: PacketMachineTemplate
-metadata:
-  name: example-packet-machine-template
-spec:
-  template:
-    spec:
-      facility: ny5
-      metro: ny
-      plan: c3.small.x86
-      billingCycle: hourly
-      project: your-packet-project-id
-      sshKeys:
-        - ssh-rsa AAAAB3...your-public-key...
-      operatingSystem: ubuntu_20_04
-      ports:
-       - name: eth1
-          bonded: false
-          layer2: true
-          ip_addresses:
-            - ipAddressReservation: "10.50.10.0/24"
-              vxlan_ids: [1001]
-              gateway: "10.50.10.1"
-
+            - address: "192.168.10.0/24"
+              vxlanId: 1000
+              gateway: "192.168.10.1"
+              subnetSize: "/32"
+            - address: "192.168.20.0/24"
+              vxlanId: 1001
+              gateway: "192.168.11.1"
+              subnetSize: "/29"
+      routes:
+        - destination: "10.40.0.0/16"
+          gateway: "192.168.10.1"
 ```
 
 ### APIs:
@@ -220,6 +277,8 @@ Requied Params : vnid (VLAN ID)
 To configure the operating system (OS), create new sub-interfaces for handling VLAN-tagged traffic, and assign IP addresses to those sub-interfaces, a user-data script is required to run at the time of OS boot.
 Below is the user-data script that would be used.
 
+// TODO(rahuls): Add route configuration in the user-data script
+
 ```sh
 #cloud-config
 package_update: true
@@ -359,7 +418,7 @@ The CAPP will use go-templates to substitute the placeholders with appropriate v
 
 ### Layer 2 Networking Setup by the CAPP Operator
 When provisioning a metal node with Layer 2 networking, the Cluster API Provider (CAPP) Operator will perform the following steps:
-1. **Create a ConfigMap for IP Address Management**: The operator will create a new ConfigMap named <cluster_name-port_name> for each port to manage IP addresses. This ConfigMap is critical for tracking and allocating IP addresses as detailed in the *IP Address Management* section.
+1. **Create a ConfigMap for IP Address Management**: The PacketMachine operator will create a new ConfigMap named <cluster_name-ip-allocations> This ConfigMap is critical for tracking and allocating IP addresses as detailed in the *IP Address Management* section.
 2. **Select an Available IP Address**: CAPP will select an available IP address from the ConfigMap to be assigned to the machine, node, or server being provisioned.
 3. **Generate User-Data Script**: Using Go templates, CAPP will substitute the necessary variables in the user-data script, such as port name, IP address, gateway, and VXLAN. These values are provided by the user through the custom resource definition.
 4. **Submit Device Creation Request**: CAPP will then submit a request to create the device, incorporating the generated user-data script for OS and network configuration.
@@ -389,11 +448,17 @@ metadata:
   name: capp-ip-allocations
   namespace: cluster-api-provider-packet-system
 Data:
-  cidr: 192.168.2.0/24
-  allocations: |
+  "da-1000-192.168.10.0/24": |
+    {
+      "machine-1": "192.168.10.2",
+      "machine-2": "192.168.10.3",
+      "machine-3": "192.168.10.4"
+    }
+  "da-1001-192.168.20.0/24": |
     {
-     "machine1": "192.168.2.2", 
-     "machine2": "192.168.2.3" 
+      "machine-1": "192.168.20.2",
+      "machine-2": "192.168.20.3",
+      "machine-3": "192.168.20.4"
     }
 ```
 

From c0b5aa8f3e4739fbb44c139e64d59d58efea1be5 Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Wed, 4 Sep 2024 13:18:38 +0530
Subject: [PATCH 11/14] changes to design doc

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 70 +++++++++++++++++++++++++----------
 1 file changed, 50 insertions(+), 20 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index 155baaa0..c25690e5 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -74,7 +74,7 @@ PacketCluster will have a new field called Networks, which will be a list of Net
 - Description: Description of the network, e.g., "Storage network for VMs" (optional)
 - IPAddresses: IP address range for the cluster network, e.g, Virtual Routing and Forwarding (VRF) . This field will be a list of strings, where each string represents an IP address range.
 - Assignment: Component responsible for allocating IP addresses to the machines, either cluster-api or dhcp.
-- Gateway: Default gateway for the network (optional)
+- Gateway: Default gateway for the network (optional) (TODO: do we need this?)
 
 **PacketCluster**
 ```go
@@ -131,9 +131,8 @@ spec:
       networks:
         - name: Storage VLAN
           description: Storage network for VMs
-          ipAddresses: ["192.168.0.0/16"]
+          ipAddresses: ["10.60.0.0/16"]
           assignment: cluster-api
-          gateway: "192.168.10.1"
 ```
 
 **PacketMachineTemplate**
@@ -185,7 +184,6 @@ type RouteSpec struct {
 ```
 
 For example:
-The following example configures the bond0 port of each node in a cluster to a hybrid bonded mode, attaches vxlanId with ID 1000 and assigns each node an IP address from range "192.168.10.0/24" with gateway 192.168.10.1
 
 ```yaml
 kind: PacketMachineTemplate
@@ -206,17 +204,39 @@ spec:
         - name: bond0
           layer2: false
           ip_addresses:
-            - address: "192.168.10.0/24"
+            - address: "10.60.10.0/24"
               vxlanId: 1000
-              gateway: "192.168.10.1"
+              gateway: "10.60.10.1"
               subnetSize: "/32"
-            - address: "192.168.20.0/24"
+      routes:
+        - destination: "10.60.0.0/16"
+          gateway: "10.60.10.1"
+
+kind: PacketMachineTemplate
+metadata:
+  name: example-packet-machine-template-1
+spec:
+  template:
+    spec:
+      facility: ny5
+      metro: ny
+      plan: c3.small.x86
+      billingCycle: hourly
+      project: your-packet-project-id
+      sshKeys:
+        - ssh-rsa AAAAB3...your-public-key...
+      operatingSystem: ubuntu_20_04
+      ports:
+        - name: bond0
+          layer2: false
+          ip_addresses:
+            - address: "10.60.20.0/24"
               vxlanId: 1001
-              gateway: "192.168.11.1"
-              subnetSize: "/29"
+              gateway: "10.60.20.1"
+              subnetSize: "/32"
       routes:
-        - destination: "10.40.0.0/16"
-          gateway: "192.168.10.1"
+        - destination: "10.60.0.0/16"
+          gateway: "10.60.20.1"
 ```
 
 ### APIs:
@@ -432,6 +452,10 @@ The send_user_state_event function in the script is responsible for sending stat
 3. **Send the Event**: The constructed JSON data is then sent to the user_state_url via a POST request. This allows the system to log the state of the network configuration process (e.g., "running," "succeeded," or "failed") along with an appropriate status code and message.
 This approach enables tracking of the server's state during the boot process, particularly for critical operations like network configuration.
 
+### Note: 
+1. If you want to connect the VLANs that are in different metros, you need to enable [Backend Transfer](https://deploy.equinix.com/developers/docs/metal/networking/backend-transfer/)
+
+2. If you want to use your own IP address range, and the VLANs are in the same metro, you can use the VRF Metal Gateway to connect the VLANs and allow CAPP to assign IP addresses from the specified range.
 
 ### IP Address Management:
 
@@ -448,19 +472,25 @@ metadata:
   name: capp-ip-allocations
   namespace: cluster-api-provider-packet-system
 Data:
-  "da-1000-192.168.10.0/24": |
+  "da-1000-10.60.10.0/24": |
     {
-      "machine-1": "192.168.10.2",
-      "machine-2": "192.168.10.3",
-      "machine-3": "192.168.10.4"
+      "machine-1": "10.60.10.2",
+      "machine-2": "10.60.10.3",
+      "machine-3": "10.60.10.4"
     }
-  "da-1001-192.168.20.0/24": |
+  "da-1001-10.60.20.0/24": |
     {
-      "machine-1": "192.168.20.2",
-      "machine-2": "192.168.20.3",
-      "machine-3": "192.168.20.4"
+      "machine-1": "10.60.20.2",
+      "machine-2": "10.60.20.3",
+      "machine-3": "10.60.20.4"
     }
 ```
 
-In the example above, capp-ip-allocations ConfigMap in the cluster-api-provider-packet-system namespace tracks IP allocations. The cidr field specifies the IP range, while the allocations field is a JSON object mapping machine names to their allocated IP addresses.
+1. In the example above, capp-ip-allocations ConfigMap in the cluster-api-provider-packet-system namespace tracks IP allocations.
+2. When a new machine is provisioned, CAPP will select an available IP address from the ConfigMap based on the specified IP range and assign it to the machine. The ConfigMap is updated to reflect the allocation, ensuring that IP addresses are not reused or double-assigned.
+3. The configmap entry is named using the format "metro-vlan-id-ip-range", where vlan-id is the VLAN ID and ip-range is the IP address range. The value is a JSON object with machine names as keys and IP addresses as values.
+4. When a machine is deleted or decommissioned, CAPP will remove the corresponding entry from the ConfigMap, making the IP address available for future allocations.
+5. The lifecycle of ConfigMap is tied to the PacketCluster, ensuring that IP address allocations are managed consistently across the cluster.
+The ConfigMap approach provides a simple and effective way to manage IP address allocations within a Kubernetes cluster, ensuring that IP addresses are tracked and assigned correctly.
+6. The lifecycle management of IP addresses is crucial for maintaining network integrity and avoiding conflicts or address exhaustion. By using ConfigMaps to track IP allocations, CAPP can efficiently manage IP addresses across multiple machines and ensure that each machine is assigned a unique and available IP address.
 

From bb70851e8813c7010f2ad0762727a9c663ec6468 Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Wed, 4 Sep 2024 16:34:56 +0530
Subject: [PATCH 12/14] changes to design doc

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index c25690e5..beacacfa 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -110,8 +110,10 @@ type NetworkSpec struct {
 }
 ```
 
-The following example configures a network named "storage VLAN" with VXLAN ID 1000, IP address range 192.168.10.0/24, and a default gateway of
-192.168.10.1. The IP addresses are assigned by the cluster-api component.
+The following example configures a network named "storage VLAN" with VXLAN ID 1000, IP address range 10.60.10.0/24, and a default gateway of
+10.60.10.1. The IP addresses are assigned to the indvidual machines by the cluster-api component.
+
+// TODO: Do we need to add the VLAN and gateway field in the NetworkSpec?
 
 ```yaml
 kind: PacketCluster
@@ -137,7 +139,7 @@ spec:
 
 **PacketMachineTemplate**
 
-To support enhanced layer2 networking capabilities, we propose adding a new Ports field under the spec of the *PacketMachineTemplate*. This field will allow users to define various network port configurations for an Equinix Metal Machine. Below is an outline of the proposed changes:
+To support enhanced layer2 networking capabilities, we propose adding a new Ports and Routes field under the spec of the *PacketMachineTemplate*. These fields will allow users to define various network port configurations and add static routes for an Equinix Metal Machine. Below is an outline of the proposed changes:
 
 ```go
 // PacketMachineSpec defines the desired state of PacketMachine.
@@ -184,6 +186,8 @@ type RouteSpec struct {
 ```
 
 For example:
+In the below example, we have defined two PacketMachineTemplates, each with a different IP address range and VLAN ID. The first template has an IP address range of 10.60.10.0/24 with a VXLAN ID of 1000, while the second template has an IP address range of 10.60.20.0/24 with a VXLAN ID of 1001. Both templates have a static route defined for the destination 10.60.0.0/24 with the gateway set to the respective gateway IP address.
+
 
 ```yaml
 kind: PacketMachineTemplate

From 3e2413ed17d074748c7dab3363925b131ab80336 Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Wed, 4 Sep 2024 16:37:54 +0530
Subject: [PATCH 13/14] minor fix

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index beacacfa..3fa81728 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -188,6 +188,7 @@ type RouteSpec struct {
 For example:
 In the below example, we have defined two PacketMachineTemplates, each with a different IP address range and VLAN ID. The first template has an IP address range of 10.60.10.0/24 with a VXLAN ID of 1000, while the second template has an IP address range of 10.60.20.0/24 with a VXLAN ID of 1001. Both templates have a static route defined for the destination 10.60.0.0/24 with the gateway set to the respective gateway IP address.
 
+Ref: https://deploy.equinix.com/developers/guides/connecting-vlans-via-vrf/
 
 ```yaml
 kind: PacketMachineTemplate

From 0de7a1a7db0356f70c6a8a6e6dcf720ffdaccd4d Mon Sep 17 00:00:00 2001
From: rahulii <r.sawra@gmail.com>
Date: Tue, 17 Sep 2024 12:01:21 +0530
Subject: [PATCH 14/14] changes discussed over last call

Signed-off-by: rahulii <r.sawra@gmail.com>
---
 docs/layer2/layer2_support.md | 51 +++++++++++++++++------------------
 1 file changed, 24 insertions(+), 27 deletions(-)

diff --git a/docs/layer2/layer2_support.md b/docs/layer2/layer2_support.md
index 3fa81728..1b31cef2 100644
--- a/docs/layer2/layer2_support.md
+++ b/docs/layer2/layer2_support.md
@@ -73,8 +73,7 @@ PacketCluster will have a new field called Networks, which will be a list of Net
 - Name: Name of the network, e.g., "storage VLAN" (optional)
 - Description: Description of the network, e.g., "Storage network for VMs" (optional)
 - IPAddresses: IP address range for the cluster network, e.g, Virtual Routing and Forwarding (VRF) . This field will be a list of strings, where each string represents an IP address range.
-- Assignment: Component responsible for allocating IP addresses to the machines, either cluster-api or dhcp.
-- Gateway: Default gateway for the network (optional) (TODO: do we need this?)
+- Assignment: Component responsible for allocating IP addresses to the machines, either cluster-api or dhcp or some other component.
 
 **PacketCluster**
 ```go
@@ -104,17 +103,12 @@ type NetworkSpec struct {
     IPAddresses []string      `json:"ipAddresses,omitempty"`
     // Assignment is component responsible for allocating IP addresses to the machines, either cluster-api or dhcp
     Assignment  AssignmentType `json:"assignment,omitempty"`
-    // Default gateway for the network
-    // +optional
-    Gateway     string        `json:"gateway,omitempty"`
 }
 ```
 
 The following example configures a network named "storage VLAN" with VXLAN ID 1000, IP address range 10.60.10.0/24, and a default gateway of
 10.60.10.1. The IP addresses are assigned to the indvidual machines by the cluster-api component.
 
-// TODO: Do we need to add the VLAN and gateway field in the NetworkSpec?
-
 ```yaml
 kind: PacketCluster
 metadata:
@@ -160,22 +154,25 @@ type Port struct {
      Bonded bool `json:"bonded,omitempty"`
      // convert port to layer 2. is false by default on new devices. changes result in /ports/id/convert/layer-[2|3] API calls
      Layer2 bool `json:"layer2"`
-     // IPAddress configurations associated with this port
-     // These are typically IP Reservations carved out of VRF.
-     IPAddresses []IPAddress `json:"ip_addresses,omitempty"`
+     // Network configurations for the port
+     Network []Network `json:"networks"`
 }
 // IPAddress represents an IP address configuration 
-type IPAddress struct {
+type Network struct {
     // Addresses to reserve for these ports.
     // for eg: can be carved out of a VRF IP Range.
     Address string `json:"address"`
-    // VLANs for EM API to find by vxlan, project, and metro match then attach to device. OS userdata template will also configure this VLAN on the bond device    
-    VXLANID int `json:"vxlanId,omitempty"`
-    // IP Address of the gateway
-    Gateway string `gateway,omitempty`
-    // Subnet Size for per machine
+    // VLANs for EM API to find by vxlan, project, and metro match then attach to device. OS userdata template will also configure this VLAN on the bond device
+    VXLAN int `json:"vxlan,omitempty"`
+    // AssignmentRange is the range of IP addresses to assign to the machine from the specified IP address range.
+    // for eg: if the IP address range is 10.60.10.0/24 , the assignment range can be '10.60.10.2-10.60.10.8'
+    // If not specified, the first available IP address from the IP address range will be assigned.
+    // This is useful when you want to reserve some IP addresses for other purposes for eg Gateways, DNS etc.
     // +optional
-    SubnetSize string `json:"subnetSize,omitempty"`
+    AssignmentRange string `json:"assignmentRange,omitempty"`
+    // AddressType is the type of address to assign to the machine. It can be either Internal or External.
+    // kubebuilder:validation:Enum=Internal;External
+    AddressType string `json:"addressType,omitempty"`
 }
 
 // RouteSpec defines the static route configuration for a PacketMachine.
@@ -208,11 +205,11 @@ spec:
       ports:
         - name: bond0
           layer2: false
-          ip_addresses:
-            - address: "10.60.10.0/24"
-              vxlanId: 1000
-              gateway: "10.60.10.1"
-              subnetSize: "/32"
+          networks:
+          - address: "10.60.10.0/24"
+            vxlan: 1000
+            assignmentRange: "10.60.10.2-10.60.10.8"
+            addressType: "Internal"
       routes:
         - destination: "10.60.0.0/16"
           gateway: "10.60.10.1"
@@ -234,11 +231,11 @@ spec:
       ports:
         - name: bond0
           layer2: false
-          ip_addresses:
-            - address: "10.60.20.0/24"
-              vxlanId: 1001
-              gateway: "10.60.20.1"
-              subnetSize: "/32"
+          networks:
+          - address: "10.60.20.0/24"
+            vxlan: 1001
+            assignmentRange: "10.60.20.2-10.60.20.8"
+            addressType: "Internal"
       routes:
         - destination: "10.60.0.0/16"
           gateway: "10.60.20.1"