IB/hfi1: Cache neighbor secure data after link up

Secure data is transferred across the link during verify
cap. This includes Neighbor Guid, Type, and Port Number.
This transfer is not guaranteed to complete until the 8051
firmware has completed processing of the state_complete
frame. Move the consumption of this data from verify cap
handling to link up handling to ensure the data is finalized.

Additionally, do not notify the SM that the link is up until
after this data is actually available.

Reviewed-by: Ira Weiny <ira.weiny@intel.com>
Reviewed-by: Easwar Hariharan <easwar.hariharan@intel.com>
Reviewed-by: Mike Marciniszyn <mike.marciniszyn@intel.com>
Signed-off-by: Stuart Summers <john.s.summers@intel.com>
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
Signed-off-by: Doug Ledford <dledford@redhat.com>

drivers/infiniband/hw/hfi1/chip.c

@@ -7323,15 +7323,6 @@ void handle_verify_cap(struct work_struct *work)
 	lcb_shutdown(dd, 0);
 	adjust_lcb_for_fpga_serdes(dd);
 
-	/*
-	 * These are now valid:
-	 *	remote VerifyCap fields in the general LNI config
-	 *	CSR DC8051_STS_REMOTE_GUID
-	 *	CSR DC8051_STS_REMOTE_NODE_TYPE
-	 *	CSR DC8051_STS_REMOTE_FM_SECURITY
-	 *	CSR DC8051_STS_REMOTE_PORT_NO
-	 */
-
 	read_vc_remote_phy(dd, &power_management, &continious);
 	read_vc_remote_fabric(dd, &vau, &z, &vcu, &vl15buf,
 			      &partner_supported_crc);
@@ -7462,20 +7453,6 @@ void handle_verify_cap(struct work_struct *work)
 	write_csr(dd, DC_LCB_ERR_EN, 0); /* mask LCB errors */
 	set_8051_lcb_access(dd);
 
-	ppd->neighbor_guid =
-		read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
-	ppd->neighbor_port_number = read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
-					DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
-	ppd->neighbor_type =
-		read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
-		DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
-	ppd->neighbor_fm_security =
-		read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) &
-		DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK;
-	dd_dev_info(dd,
-		    "Neighbor Guid: %llx Neighbor type %d MgmtAllowed %d FM security bypass %d\n",
-		    ppd->neighbor_guid, ppd->neighbor_type,
-		    ppd->mgmt_allowed, ppd->neighbor_fm_security);
 	if (ppd->mgmt_allowed)
 		add_full_mgmt_pkey(ppd);
 
@@ -10535,11 +10512,8 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)
 			goto unexpected;
 		}
 
-		ppd->host_link_state = HLS_UP_INIT;
 		ret = wait_logical_linkstate(ppd, IB_PORT_INIT, 1000);
 		if (ret) {
-			/* logical state didn't change, stay at going_up */
-			ppd->host_link_state = HLS_GOING_UP;
 			dd_dev_err(dd,
 				   "%s: logical state did not change to INIT\n",
 				   __func__);
@@ -10553,6 +10527,7 @@ int set_link_state(struct hfi1_pportdata *ppd, u32 state)
 			add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
 
 			handle_linkup_change(dd, 1);
+			ppd->host_link_state = HLS_UP_INIT;
 		}
 		break;
 	case HLS_UP_ARMED:

drivers/infiniband/hw/hfi1/hfi.h

@@ -1307,7 +1307,16 @@ int hfi1_reset_device(int);
 /* return the driver's idea of the logical OPA port state */
 static inline u32 driver_lstate(struct hfi1_pportdata *ppd)
 {
-	return ppd->lstate; /* use the cached value */
+	/*
+	 * The driver does some processing from the time the logical
+	 * link state is at INIT to the time the SM can be notified
+	 * as such. Return IB_PORT_DOWN until the software state
+	 * is ready.
+	 */
+	if (ppd->lstate == IB_PORT_INIT && !(ppd->host_link_state & HLS_UP))
+		return IB_PORT_DOWN;
+	else
+		return ppd->lstate;
 }
 
 void receive_interrupt_work(struct work_struct *work);

drivers/infiniband/hw/hfi1/intr.c

@@ -131,19 +131,24 @@ void handle_linkup_change(struct hfi1_devdata *dd, u32 linkup)
 		if (quick_linkup || dd->icode == ICODE_FUNCTIONAL_SIMULATOR) {
 			set_up_vl15(dd, dd->vau, dd->vl15_init);
 			assign_remote_cm_au_table(dd, dd->vcu);
-			ppd->neighbor_guid =
-				read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
-			ppd->neighbor_type =
-				read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
-					DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
-			ppd->neighbor_port_number =
-				read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
-					 DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
-			dd_dev_info(dd, "Neighbor GUID: %llx Neighbor type %d\n",
-				    ppd->neighbor_guid,
-				    ppd->neighbor_type);
 		}
 
+		ppd->neighbor_guid =
+			read_csr(dd, DC_DC8051_STS_REMOTE_GUID);
+		ppd->neighbor_type =
+			read_csr(dd, DC_DC8051_STS_REMOTE_NODE_TYPE) &
+				 DC_DC8051_STS_REMOTE_NODE_TYPE_VAL_MASK;
+		ppd->neighbor_port_number =
+			read_csr(dd, DC_DC8051_STS_REMOTE_PORT_NO) &
+				 DC_DC8051_STS_REMOTE_PORT_NO_VAL_SMASK;
+		ppd->neighbor_fm_security =
+			read_csr(dd, DC_DC8051_STS_REMOTE_FM_SECURITY) &
+				 DC_DC8051_STS_LOCAL_FM_SECURITY_DISABLED_MASK;
+		dd_dev_info(dd,
+			    "Neighbor Guid %llx, Type %d, Port Num %d\n",
+			    ppd->neighbor_guid, ppd->neighbor_type,
+			    ppd->neighbor_port_number);
+
 		/* physical link went up */
 		ppd->linkup = 1;
 		ppd->offline_disabled_reason =