|
@@ -196,37 +196,85 @@ xpc_activate_IRQ_send_local_sn2(int from_nasid)
|
|
|
wake_up_interruptible(&xpc_activate_IRQ_wq);
|
|
|
}
|
|
|
|
|
|
-static void
|
|
|
-xpc_IPI_send_local_activate_sn2(int from_nasid)
|
|
|
-{
|
|
|
- xpc_activate_IRQ_send_local_sn2(from_nasid);
|
|
|
-}
|
|
|
+/*
|
|
|
+ * IPIs associated with SGI_XPC_NOTIFY IRQ.
|
|
|
+ */
|
|
|
|
|
|
+/*
|
|
|
+ * Check to see if there is any channel activity to/from the specified
|
|
|
+ * partition.
|
|
|
+ */
|
|
|
static void
|
|
|
-xpc_IPI_send_activated_sn2(struct xpc_partition *part)
|
|
|
+xpc_check_for_channel_activity_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
- xpc_activate_IRQ_send_sn2(part->remote_amos_page_pa,
|
|
|
- cnodeid_to_nasid(0), part->remote_act_nasid,
|
|
|
- part->remote_act_phys_cpuid);
|
|
|
-}
|
|
|
+ u64 IPI_amo;
|
|
|
+ unsigned long irq_flags;
|
|
|
|
|
|
-static void
|
|
|
-xpc_IPI_send_local_reactivate_sn2(int from_nasid)
|
|
|
-{
|
|
|
- xpc_activate_IRQ_send_local_sn2(from_nasid);
|
|
|
+ IPI_amo = xpc_IPI_receive_sn2(part->sn.sn2.local_IPI_amo_va);
|
|
|
+ if (IPI_amo == 0)
|
|
|
+ return;
|
|
|
+
|
|
|
+ spin_lock_irqsave(&part->IPI_lock, irq_flags);
|
|
|
+ part->local_IPI_amo |= IPI_amo;
|
|
|
+ spin_unlock_irqrestore(&part->IPI_lock, irq_flags);
|
|
|
+
|
|
|
+ dev_dbg(xpc_chan, "received IPI from partid=%d, IPI_amo=0x%lx\n",
|
|
|
+ XPC_PARTID(part), IPI_amo);
|
|
|
+
|
|
|
+ xpc_wakeup_channel_mgr(part);
|
|
|
}
|
|
|
|
|
|
-static void
|
|
|
-xpc_IPI_send_disengage_sn2(struct xpc_partition *part)
|
|
|
+/*
|
|
|
+ * Handle the receipt of a SGI_XPC_NOTIFY IRQ by seeing whether the specified
|
|
|
+ * partition actually sent it. Since SGI_XPC_NOTIFY IRQs may be shared by more
|
|
|
+ * than one partition, we use an AMO_t structure per partition to indicate
|
|
|
+ * whether a partition has sent an IPI or not. If it has, then wake up the
|
|
|
+ * associated kthread to handle it.
|
|
|
+ *
|
|
|
+ * All SGI_XPC_NOTIFY IRQs received by XPC are the result of IPIs sent by XPC
|
|
|
+ * running on other partitions.
|
|
|
+ *
|
|
|
+ * Noteworthy Arguments:
|
|
|
+ *
|
|
|
+ * irq - Interrupt ReQuest number. NOT USED.
|
|
|
+ *
|
|
|
+ * dev_id - partid of IPI's potential sender.
|
|
|
+ */
|
|
|
+static irqreturn_t
|
|
|
+xpc_handle_notify_IRQ_sn2(int irq, void *dev_id)
|
|
|
{
|
|
|
- xpc_activate_IRQ_send_sn2(part->remote_amos_page_pa,
|
|
|
- cnodeid_to_nasid(0), part->remote_act_nasid,
|
|
|
- part->remote_act_phys_cpuid);
|
|
|
+ short partid = (short)(u64)dev_id;
|
|
|
+ struct xpc_partition *part = &xpc_partitions[partid];
|
|
|
+
|
|
|
+ DBUG_ON(partid < 0 || partid >= xp_max_npartitions);
|
|
|
+
|
|
|
+ if (xpc_part_ref(part)) {
|
|
|
+ xpc_check_for_channel_activity_sn2(part);
|
|
|
+
|
|
|
+ xpc_part_deref(part);
|
|
|
+ }
|
|
|
+ return IRQ_HANDLED;
|
|
|
}
|
|
|
|
|
|
/*
|
|
|
- * IPIs associated with SGI_XPC_NOTIFY IRQ.
|
|
|
+ * Check to see if xpc_handle_notify_IRQ_sn2() dropped any IPIs on the floor
|
|
|
+ * because the write to their associated IPI amo completed after the IRQ/IPI
|
|
|
+ * was received.
|
|
|
*/
|
|
|
+static void
|
|
|
+xpc_dropped_notify_IRQ_check_sn2(struct xpc_partition *part)
|
|
|
+{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
+
|
|
|
+ if (xpc_part_ref(part)) {
|
|
|
+ xpc_check_for_channel_activity_sn2(part);
|
|
|
+
|
|
|
+ part_sn2->dropped_notify_IRQ_timer.expires = jiffies +
|
|
|
+ XPC_P_DROPPED_IPI_WAIT_INTERVAL;
|
|
|
+ add_timer(&part_sn2->dropped_notify_IRQ_timer);
|
|
|
+ xpc_part_deref(part);
|
|
|
+ }
|
|
|
+}
|
|
|
|
|
|
/*
|
|
|
* Send an IPI to the remote partition that is associated with the
|
|
@@ -237,13 +285,14 @@ xpc_notify_IRQ_send_sn2(struct xpc_channel *ch, u8 ipi_flag,
|
|
|
char *ipi_flag_string, unsigned long *irq_flags)
|
|
|
{
|
|
|
struct xpc_partition *part = &xpc_partitions[ch->partid];
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
enum xp_retval ret;
|
|
|
|
|
|
if (likely(part->act_state != XPC_P_DEACTIVATING)) {
|
|
|
- ret = xpc_IPI_send_sn2(part->remote_IPI_amo_va,
|
|
|
+ ret = xpc_IPI_send_sn2(part_sn2->remote_IPI_amo_va,
|
|
|
(u64)ipi_flag << (ch->number * 8),
|
|
|
- part->remote_IPI_nasid,
|
|
|
- part->remote_IPI_phys_cpuid,
|
|
|
+ part_sn2->remote_IPI_nasid,
|
|
|
+ part_sn2->remote_IPI_phys_cpuid,
|
|
|
SGI_XPC_NOTIFY);
|
|
|
dev_dbg(xpc_chan, "%s sent to partid=%d, channel=%d, ret=%d\n",
|
|
|
ipi_flag_string, ch->partid, ch->number, ret);
|
|
@@ -263,7 +312,7 @@ xpc_notify_IRQ_send_sn2(struct xpc_channel *ch, u8 ipi_flag,
|
|
|
/*
|
|
|
* Make it look like the remote partition, which is associated with the
|
|
|
* specified channel, sent us an IPI. This faked IPI will be handled
|
|
|
- * by xpc_dropped_IPI_check().
|
|
|
+ * by xpc_dropped_notify_IRQ_check_sn2().
|
|
|
*/
|
|
|
static void
|
|
|
xpc_notify_IRQ_send_local_sn2(struct xpc_channel *ch, u8 ipi_flag,
|
|
@@ -271,7 +320,7 @@ xpc_notify_IRQ_send_local_sn2(struct xpc_channel *ch, u8 ipi_flag,
|
|
|
{
|
|
|
struct xpc_partition *part = &xpc_partitions[ch->partid];
|
|
|
|
|
|
- FETCHOP_STORE_OP(TO_AMO((u64)&part->local_IPI_amo_va->variable),
|
|
|
+ FETCHOP_STORE_OP(TO_AMO((u64)&part->sn.sn2.local_IPI_amo_va->variable),
|
|
|
FETCHOP_OR, ((u64)ipi_flag << (ch->number * 8)));
|
|
|
dev_dbg(xpc_chan, "%s sent local from partid=%d, channel=%d\n",
|
|
|
ipi_flag_string, ch->partid, ch->number);
|
|
@@ -281,7 +330,8 @@ xpc_notify_IRQ_send_local_sn2(struct xpc_channel *ch, u8 ipi_flag,
|
|
|
xpc_notify_IRQ_send_local_sn2(_ch, _ipi_f, #_ipi_f)
|
|
|
|
|
|
static void
|
|
|
-xpc_IPI_send_closerequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
|
|
|
+xpc_send_channel_closerequest_sn2(struct xpc_channel *ch,
|
|
|
+ unsigned long *irq_flags)
|
|
|
{
|
|
|
struct xpc_openclose_args *args = ch->local_openclose_args;
|
|
|
|
|
@@ -290,13 +340,15 @@ xpc_IPI_send_closerequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-xpc_IPI_send_closereply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
|
|
|
+xpc_send_channel_closereply_sn2(struct xpc_channel *ch,
|
|
|
+ unsigned long *irq_flags)
|
|
|
{
|
|
|
XPC_NOTIFY_IRQ_SEND_SN2(ch, XPC_IPI_CLOSEREPLY, irq_flags);
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-xpc_IPI_send_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
|
|
|
+xpc_send_channel_openrequest_sn2(struct xpc_channel *ch,
|
|
|
+ unsigned long *irq_flags)
|
|
|
{
|
|
|
struct xpc_openclose_args *args = ch->local_openclose_args;
|
|
|
|
|
@@ -306,7 +358,7 @@ xpc_IPI_send_openrequest_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-xpc_IPI_send_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
|
|
|
+xpc_send_channel_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
|
|
|
{
|
|
|
struct xpc_openclose_args *args = ch->local_openclose_args;
|
|
|
|
|
@@ -317,13 +369,13 @@ xpc_IPI_send_openreply_sn2(struct xpc_channel *ch, unsigned long *irq_flags)
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-xpc_IPI_send_msgrequest_sn2(struct xpc_channel *ch)
|
|
|
+xpc_send_channel_msgrequest_sn2(struct xpc_channel *ch)
|
|
|
{
|
|
|
XPC_NOTIFY_IRQ_SEND_SN2(ch, XPC_IPI_MSGREQUEST, NULL);
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-xpc_IPI_send_local_msgrequest_sn2(struct xpc_channel *ch)
|
|
|
+xpc_send_channel_local_msgrequest_sn2(struct xpc_channel *ch)
|
|
|
{
|
|
|
XPC_NOTIFY_IRQ_SEND_LOCAL_SN2(ch, XPC_IPI_MSGREQUEST);
|
|
|
}
|
|
@@ -334,10 +386,10 @@ xpc_IPI_send_local_msgrequest_sn2(struct xpc_channel *ch)
|
|
|
*/
|
|
|
|
|
|
static void
|
|
|
-xpc_mark_partition_engaged_sn2(struct xpc_partition *part)
|
|
|
+xpc_indicate_partition_engaged_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
unsigned long irq_flags;
|
|
|
- AMO_t *amo = (AMO_t *)__va(part->remote_amos_page_pa +
|
|
|
+ AMO_t *amo = (AMO_t *)__va(part->sn.sn2.remote_amos_page_pa +
|
|
|
(XPC_ENGAGED_PARTITIONS_AMO *
|
|
|
sizeof(AMO_t)));
|
|
|
|
|
@@ -360,10 +412,11 @@ xpc_mark_partition_engaged_sn2(struct xpc_partition *part)
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-xpc_mark_partition_disengaged_sn2(struct xpc_partition *part)
|
|
|
+xpc_indicate_partition_disengaged_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
unsigned long irq_flags;
|
|
|
- AMO_t *amo = (AMO_t *)__va(part->remote_amos_page_pa +
|
|
|
+ AMO_t *amo = (AMO_t *)__va(part_sn2->remote_amos_page_pa +
|
|
|
(XPC_ENGAGED_PARTITIONS_AMO *
|
|
|
sizeof(AMO_t)));
|
|
|
|
|
@@ -383,96 +436,44 @@ xpc_mark_partition_disengaged_sn2(struct xpc_partition *part)
|
|
|
xp_nofault_PIOR_target));
|
|
|
|
|
|
local_irq_restore(irq_flags);
|
|
|
-}
|
|
|
-
|
|
|
-static void
|
|
|
-xpc_request_partition_disengage_sn2(struct xpc_partition *part)
|
|
|
-{
|
|
|
- unsigned long irq_flags;
|
|
|
- AMO_t *amo = (AMO_t *)__va(part->remote_amos_page_pa +
|
|
|
- (XPC_DISENGAGE_REQUEST_AMO * sizeof(AMO_t)));
|
|
|
-
|
|
|
- local_irq_save(irq_flags);
|
|
|
|
|
|
- /* set bit corresponding to our partid in remote partition's AMO */
|
|
|
- FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
|
|
|
- (1UL << sn_partition_id));
|
|
|
/*
|
|
|
- * We must always use the nofault function regardless of whether we
|
|
|
- * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
|
|
|
- * didn't, we'd never know that the other partition is down and would
|
|
|
- * keep sending IPIs and AMOs to it until the heartbeat times out.
|
|
|
+ * Send activate IRQ to get other side to see that we've cleared our
|
|
|
+ * bit in their engaged partitions AMO.
|
|
|
*/
|
|
|
- (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
|
|
|
- variable),
|
|
|
- xp_nofault_PIOR_target));
|
|
|
-
|
|
|
- local_irq_restore(irq_flags);
|
|
|
+ xpc_activate_IRQ_send_sn2(part_sn2->remote_amos_page_pa,
|
|
|
+ cnodeid_to_nasid(0),
|
|
|
+ part_sn2->activate_IRQ_nasid,
|
|
|
+ part_sn2->activate_IRQ_phys_cpuid);
|
|
|
}
|
|
|
|
|
|
-static void
|
|
|
-xpc_cancel_partition_disengage_request_sn2(struct xpc_partition *part)
|
|
|
-{
|
|
|
- unsigned long irq_flags;
|
|
|
- AMO_t *amo = (AMO_t *)__va(part->remote_amos_page_pa +
|
|
|
- (XPC_DISENGAGE_REQUEST_AMO * sizeof(AMO_t)));
|
|
|
-
|
|
|
- local_irq_save(irq_flags);
|
|
|
-
|
|
|
- /* clear bit corresponding to our partid in remote partition's AMO */
|
|
|
- FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
|
|
|
- ~(1UL << sn_partition_id));
|
|
|
- /*
|
|
|
- * We must always use the nofault function regardless of whether we
|
|
|
- * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
|
|
|
- * didn't, we'd never know that the other partition is down and would
|
|
|
- * keep sending IPIs and AMOs to it until the heartbeat times out.
|
|
|
- */
|
|
|
- (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
|
|
|
- variable),
|
|
|
- xp_nofault_PIOR_target));
|
|
|
-
|
|
|
- local_irq_restore(irq_flags);
|
|
|
-}
|
|
|
-
|
|
|
-static u64
|
|
|
-xpc_partition_engaged_sn2(u64 partid_mask)
|
|
|
+static int
|
|
|
+xpc_partition_engaged_sn2(short partid)
|
|
|
{
|
|
|
AMO_t *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO;
|
|
|
|
|
|
- /* return our partition's AMO variable ANDed with partid_mask */
|
|
|
+ /* our partition's AMO variable ANDed with partid mask */
|
|
|
return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
|
|
|
- partid_mask);
|
|
|
+ (1UL << partid)) != 0;
|
|
|
}
|
|
|
|
|
|
-static u64
|
|
|
-xpc_partition_disengage_requested_sn2(u64 partid_mask)
|
|
|
-{
|
|
|
- AMO_t *amo = xpc_vars->amos_page + XPC_DISENGAGE_REQUEST_AMO;
|
|
|
-
|
|
|
- /* return our partition's AMO variable ANDed with partid_mask */
|
|
|
- return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
|
|
|
- partid_mask);
|
|
|
-}
|
|
|
-
|
|
|
-static void
|
|
|
-xpc_clear_partition_engaged_sn2(u64 partid_mask)
|
|
|
+static int
|
|
|
+xpc_any_partition_engaged_sn2(void)
|
|
|
{
|
|
|
AMO_t *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO;
|
|
|
|
|
|
- /* clear bit(s) based on partid_mask in our partition's AMO */
|
|
|
- FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
|
|
|
- ~partid_mask);
|
|
|
+ /* our partition's AMO variable */
|
|
|
+ return FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) != 0;
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-xpc_clear_partition_disengage_request_sn2(u64 partid_mask)
|
|
|
+xpc_assume_partition_disengaged_sn2(short partid)
|
|
|
{
|
|
|
- AMO_t *amo = xpc_vars->amos_page + XPC_DISENGAGE_REQUEST_AMO;
|
|
|
+ AMO_t *amo = xpc_vars->amos_page + XPC_ENGAGED_PARTITIONS_AMO;
|
|
|
|
|
|
- /* clear bit(s) based on partid_mask in our partition's AMO */
|
|
|
+ /* clear bit(s) based on partid mask in our partition's AMO */
|
|
|
FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
|
|
|
- ~partid_mask);
|
|
|
+ ~(1UL << partid));
|
|
|
}
|
|
|
|
|
|
/* original protection values for each node */
|
|
@@ -545,7 +546,6 @@ xpc_rsvd_page_init_sn2(struct xpc_rsvd_page *rp)
|
|
|
xpc_vars_part = (struct xpc_vars_part_sn2 *)((u8 *)XPC_RP_VARS(rp) +
|
|
|
XPC_RP_VARS_SIZE);
|
|
|
|
|
|
-
|
|
|
/*
|
|
|
* Before clearing xpc_vars, see if a page of AMOs had been previously
|
|
|
* allocated. If not we'll need to allocate one and set permissions
|
|
@@ -583,8 +583,8 @@ xpc_rsvd_page_init_sn2(struct xpc_rsvd_page *rp)
|
|
|
memset(xpc_vars, 0, sizeof(struct xpc_vars_sn2));
|
|
|
|
|
|
xpc_vars->version = XPC_V_VERSION;
|
|
|
- xpc_vars->act_nasid = cpuid_to_nasid(0);
|
|
|
- xpc_vars->act_phys_cpuid = cpu_physical_id(0);
|
|
|
+ xpc_vars->activate_IRQ_nasid = cpuid_to_nasid(0);
|
|
|
+ xpc_vars->activate_IRQ_phys_cpuid = cpu_physical_id(0);
|
|
|
xpc_vars->vars_part_pa = __pa(xpc_vars_part);
|
|
|
xpc_vars->amos_page_pa = ia64_tpa((u64)amos_page);
|
|
|
xpc_vars->amos_page = amos_page; /* save for next load of XPC */
|
|
@@ -599,7 +599,7 @@ xpc_rsvd_page_init_sn2(struct xpc_rsvd_page *rp)
|
|
|
|
|
|
/* initialize the engaged remote partitions related AMO variables */
|
|
|
(void)xpc_IPI_init_sn2(XPC_ENGAGED_PARTITIONS_AMO);
|
|
|
- (void)xpc_IPI_init_sn2(XPC_DISENGAGE_REQUEST_AMO);
|
|
|
+ (void)xpc_IPI_init_sn2(XPC_DEACTIVATE_REQUEST_AMO);
|
|
|
|
|
|
return xpSuccess;
|
|
|
}
|
|
@@ -671,7 +671,7 @@ xpc_check_remote_hb_sn2(void)
|
|
|
|
|
|
/* pull the remote_hb cache line */
|
|
|
ret = xp_remote_memcpy(remote_vars,
|
|
|
- (void *)part->remote_vars_pa,
|
|
|
+ (void *)part->sn.sn2.remote_vars_pa,
|
|
|
XPC_RP_VARS_SIZE);
|
|
|
if (ret != xpSuccess) {
|
|
|
XPC_DEACTIVATE_PARTITION(part, ret);
|
|
@@ -726,10 +726,86 @@ xpc_get_remote_vars_sn2(u64 remote_vars_pa, struct xpc_vars_sn2 *remote_vars)
|
|
|
}
|
|
|
|
|
|
static void
|
|
|
-xpc_initiate_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
|
|
|
- u64 remote_rp_pa, int nasid)
|
|
|
+xpc_request_partition_activation_sn2(struct xpc_rsvd_page *remote_rp,
|
|
|
+ u64 remote_rp_pa, int nasid)
|
|
|
{
|
|
|
- xpc_IPI_send_local_activate(nasid);
|
|
|
+ xpc_activate_IRQ_send_local_sn2(nasid);
|
|
|
+}
|
|
|
+
|
|
|
+static void
|
|
|
+xpc_request_partition_reactivation_sn2(struct xpc_partition *part)
|
|
|
+{
|
|
|
+ xpc_activate_IRQ_send_local_sn2(part->sn.sn2.activate_IRQ_nasid);
|
|
|
+}
|
|
|
+
|
|
|
+static void
|
|
|
+xpc_request_partition_deactivation_sn2(struct xpc_partition *part)
|
|
|
+{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
+ unsigned long irq_flags;
|
|
|
+ AMO_t *amo = (AMO_t *)__va(part_sn2->remote_amos_page_pa +
|
|
|
+ (XPC_DEACTIVATE_REQUEST_AMO * sizeof(AMO_t)));
|
|
|
+
|
|
|
+ local_irq_save(irq_flags);
|
|
|
+
|
|
|
+ /* set bit corresponding to our partid in remote partition's AMO */
|
|
|
+ FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_OR,
|
|
|
+ (1UL << sn_partition_id));
|
|
|
+ /*
|
|
|
+ * We must always use the nofault function regardless of whether we
|
|
|
+ * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
|
|
|
+ * didn't, we'd never know that the other partition is down and would
|
|
|
+ * keep sending IPIs and AMOs to it until the heartbeat times out.
|
|
|
+ */
|
|
|
+ (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
|
|
|
+ variable),
|
|
|
+ xp_nofault_PIOR_target));
|
|
|
+
|
|
|
+ local_irq_restore(irq_flags);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Send activate IRQ to get other side to see that we've set our
|
|
|
+ * bit in their deactivate request AMO.
|
|
|
+ */
|
|
|
+ xpc_activate_IRQ_send_sn2(part_sn2->remote_amos_page_pa,
|
|
|
+ cnodeid_to_nasid(0),
|
|
|
+ part_sn2->activate_IRQ_nasid,
|
|
|
+ part_sn2->activate_IRQ_phys_cpuid);
|
|
|
+}
|
|
|
+
|
|
|
+static void
|
|
|
+xpc_cancel_partition_deactivation_request_sn2(struct xpc_partition *part)
|
|
|
+{
|
|
|
+ unsigned long irq_flags;
|
|
|
+ AMO_t *amo = (AMO_t *)__va(part->sn.sn2.remote_amos_page_pa +
|
|
|
+ (XPC_DEACTIVATE_REQUEST_AMO * sizeof(AMO_t)));
|
|
|
+
|
|
|
+ local_irq_save(irq_flags);
|
|
|
+
|
|
|
+ /* clear bit corresponding to our partid in remote partition's AMO */
|
|
|
+ FETCHOP_STORE_OP(TO_AMO((u64)&amo->variable), FETCHOP_AND,
|
|
|
+ ~(1UL << sn_partition_id));
|
|
|
+ /*
|
|
|
+ * We must always use the nofault function regardless of whether we
|
|
|
+ * are on a Shub 1.1 system or a Shub 1.2 slice 0xc processor. If we
|
|
|
+ * didn't, we'd never know that the other partition is down and would
|
|
|
+ * keep sending IPIs and AMOs to it until the heartbeat times out.
|
|
|
+ */
|
|
|
+ (void)xp_nofault_PIOR((u64 *)GLOBAL_MMR_ADDR(NASID_GET(&amo->
|
|
|
+ variable),
|
|
|
+ xp_nofault_PIOR_target));
|
|
|
+
|
|
|
+ local_irq_restore(irq_flags);
|
|
|
+}
|
|
|
+
|
|
|
+static int
|
|
|
+xpc_partition_deactivation_requested_sn2(short partid)
|
|
|
+{
|
|
|
+ AMO_t *amo = xpc_vars->amos_page + XPC_DEACTIVATE_REQUEST_AMO;
|
|
|
+
|
|
|
+ /* our partition's AMO variable ANDed with partid mask */
|
|
|
+ return (FETCHOP_LOAD_OP(TO_AMO((u64)&amo->variable), FETCHOP_LOAD) &
|
|
|
+ (1UL << partid)) != 0;
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -741,6 +817,8 @@ xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
|
|
|
u64 remote_vars_pa,
|
|
|
struct xpc_vars_sn2 *remote_vars)
|
|
|
{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
+
|
|
|
part->remote_rp_version = remote_rp_version;
|
|
|
dev_dbg(xpc_part, " remote_rp_version = 0x%016x\n",
|
|
|
part->remote_rp_version);
|
|
@@ -752,33 +830,34 @@ xpc_update_partition_info_sn2(struct xpc_partition *part, u8 remote_rp_version,
|
|
|
part->remote_rp_pa = remote_rp_pa;
|
|
|
dev_dbg(xpc_part, " remote_rp_pa = 0x%016lx\n", part->remote_rp_pa);
|
|
|
|
|
|
- part->remote_vars_pa = remote_vars_pa;
|
|
|
+ part_sn2->remote_vars_pa = remote_vars_pa;
|
|
|
dev_dbg(xpc_part, " remote_vars_pa = 0x%016lx\n",
|
|
|
- part->remote_vars_pa);
|
|
|
+ part_sn2->remote_vars_pa);
|
|
|
|
|
|
part->last_heartbeat = remote_vars->heartbeat;
|
|
|
dev_dbg(xpc_part, " last_heartbeat = 0x%016lx\n",
|
|
|
part->last_heartbeat);
|
|
|
|
|
|
- part->remote_vars_part_pa = remote_vars->vars_part_pa;
|
|
|
+ part_sn2->remote_vars_part_pa = remote_vars->vars_part_pa;
|
|
|
dev_dbg(xpc_part, " remote_vars_part_pa = 0x%016lx\n",
|
|
|
- part->remote_vars_part_pa);
|
|
|
+ part_sn2->remote_vars_part_pa);
|
|
|
|
|
|
- part->remote_act_nasid = remote_vars->act_nasid;
|
|
|
- dev_dbg(xpc_part, " remote_act_nasid = 0x%x\n",
|
|
|
- part->remote_act_nasid);
|
|
|
+ part_sn2->activate_IRQ_nasid = remote_vars->activate_IRQ_nasid;
|
|
|
+ dev_dbg(xpc_part, " activate_IRQ_nasid = 0x%x\n",
|
|
|
+ part_sn2->activate_IRQ_nasid);
|
|
|
|
|
|
- part->remote_act_phys_cpuid = remote_vars->act_phys_cpuid;
|
|
|
- dev_dbg(xpc_part, " remote_act_phys_cpuid = 0x%x\n",
|
|
|
- part->remote_act_phys_cpuid);
|
|
|
+ part_sn2->activate_IRQ_phys_cpuid =
|
|
|
+ remote_vars->activate_IRQ_phys_cpuid;
|
|
|
+ dev_dbg(xpc_part, " activate_IRQ_phys_cpuid = 0x%x\n",
|
|
|
+ part_sn2->activate_IRQ_phys_cpuid);
|
|
|
|
|
|
- part->remote_amos_page_pa = remote_vars->amos_page_pa;
|
|
|
+ part_sn2->remote_amos_page_pa = remote_vars->amos_page_pa;
|
|
|
dev_dbg(xpc_part, " remote_amos_page_pa = 0x%lx\n",
|
|
|
- part->remote_amos_page_pa);
|
|
|
+ part_sn2->remote_amos_page_pa);
|
|
|
|
|
|
- part->remote_vars_version = remote_vars->version;
|
|
|
+ part_sn2->remote_vars_version = remote_vars->version;
|
|
|
dev_dbg(xpc_part, " remote_vars_version = 0x%x\n",
|
|
|
- part->remote_vars_version);
|
|
|
+ part_sn2->remote_vars_version);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -807,6 +886,7 @@ xpc_identify_activate_IRQ_req_sn2(int nasid)
|
|
|
unsigned long remote_rp_stamp = 0;
|
|
|
short partid;
|
|
|
struct xpc_partition *part;
|
|
|
+ struct xpc_partition_sn2 *part_sn2;
|
|
|
enum xp_retval ret;
|
|
|
|
|
|
/* pull over the reserved page structure */
|
|
@@ -822,11 +902,11 @@ xpc_identify_activate_IRQ_req_sn2(int nasid)
|
|
|
|
|
|
remote_vars_pa = remote_rp->sn.vars_pa;
|
|
|
remote_rp_version = remote_rp->version;
|
|
|
- if (XPC_SUPPORTS_RP_STAMP(remote_rp_version))
|
|
|
- remote_rp_stamp = remote_rp->stamp;
|
|
|
+ remote_rp_stamp = remote_rp->stamp;
|
|
|
|
|
|
partid = remote_rp->SAL_partid;
|
|
|
part = &xpc_partitions[partid];
|
|
|
+ part_sn2 = &part->sn.sn2;
|
|
|
|
|
|
/* pull over the cross partition variables */
|
|
|
|
|
@@ -834,7 +914,6 @@ xpc_identify_activate_IRQ_req_sn2(int nasid)
|
|
|
|
|
|
ret = xpc_get_remote_vars_sn2(remote_vars_pa, remote_vars);
|
|
|
if (ret != xpSuccess) {
|
|
|
-
|
|
|
dev_warn(xpc_part, "unable to get XPC variables from nasid %d, "
|
|
|
"which sent interrupt, reason=%d\n", nasid, ret);
|
|
|
|
|
@@ -855,18 +934,12 @@ xpc_identify_activate_IRQ_req_sn2(int nasid)
|
|
|
&remote_rp_stamp, remote_rp_pa,
|
|
|
remote_vars_pa, remote_vars);
|
|
|
|
|
|
- if (XPC_SUPPORTS_DISENGAGE_REQUEST(part->remote_vars_version)) {
|
|
|
- if (xpc_partition_disengage_requested(1UL << partid)) {
|
|
|
- /*
|
|
|
- * Other side is waiting on us to disengage,
|
|
|
- * even though we already have.
|
|
|
- */
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
- } else {
|
|
|
- /* other side doesn't support disengage requests */
|
|
|
- xpc_clear_partition_disengage_request(1UL << partid);
|
|
|
+ if (xpc_partition_deactivation_requested_sn2(partid)) {
|
|
|
+ /*
|
|
|
+ * Other side is waiting on us to deactivate even though
|
|
|
+ * we already have.
|
|
|
+ */
|
|
|
+ return;
|
|
|
}
|
|
|
|
|
|
xpc_activate_partition(part);
|
|
@@ -874,93 +947,30 @@ xpc_identify_activate_IRQ_req_sn2(int nasid)
|
|
|
}
|
|
|
|
|
|
DBUG_ON(part->remote_rp_version == 0);
|
|
|
- DBUG_ON(part->remote_vars_version == 0);
|
|
|
-
|
|
|
- if (!XPC_SUPPORTS_RP_STAMP(part->remote_rp_version)) {
|
|
|
- DBUG_ON(XPC_SUPPORTS_DISENGAGE_REQUEST(part->
|
|
|
- remote_vars_version));
|
|
|
-
|
|
|
- if (!XPC_SUPPORTS_RP_STAMP(remote_rp_version)) {
|
|
|
- DBUG_ON(XPC_SUPPORTS_DISENGAGE_REQUEST(remote_vars->
|
|
|
- version));
|
|
|
- /* see if the other side rebooted */
|
|
|
- if (part->remote_amos_page_pa ==
|
|
|
- remote_vars->amos_page_pa &&
|
|
|
- xpc_hb_allowed(sn_partition_id,
|
|
|
- &remote_vars->heartbeating_to_mask)) {
|
|
|
- /* doesn't look that way, so ignore the IPI */
|
|
|
- return;
|
|
|
- }
|
|
|
- }
|
|
|
+ DBUG_ON(part_sn2->remote_vars_version == 0);
|
|
|
|
|
|
- /*
|
|
|
- * Other side rebooted and previous XPC didn't support the
|
|
|
- * disengage request, so we don't need to do anything special.
|
|
|
- */
|
|
|
+ if (remote_rp_stamp != part->remote_rp_stamp) {
|
|
|
|
|
|
- xpc_update_partition_info_sn2(part, remote_rp_version,
|
|
|
- &remote_rp_stamp, remote_rp_pa,
|
|
|
- remote_vars_pa, remote_vars);
|
|
|
- part->reactivate_nasid = nasid;
|
|
|
- XPC_DEACTIVATE_PARTITION(part, xpReactivating);
|
|
|
- return;
|
|
|
- }
|
|
|
+ /* the other side rebooted */
|
|
|
|
|
|
- DBUG_ON(!XPC_SUPPORTS_DISENGAGE_REQUEST(part->remote_vars_version));
|
|
|
-
|
|
|
- if (!XPC_SUPPORTS_RP_STAMP(remote_rp_version)) {
|
|
|
- DBUG_ON(!XPC_SUPPORTS_DISENGAGE_REQUEST(remote_vars->version));
|
|
|
-
|
|
|
- /*
|
|
|
- * Other side rebooted and previous XPC did support the
|
|
|
- * disengage request, but the new one doesn't.
|
|
|
- */
|
|
|
-
|
|
|
- xpc_clear_partition_engaged(1UL << partid);
|
|
|
- xpc_clear_partition_disengage_request(1UL << partid);
|
|
|
+ DBUG_ON(xpc_partition_engaged_sn2(partid));
|
|
|
+ DBUG_ON(xpc_partition_deactivation_requested_sn2(partid));
|
|
|
|
|
|
xpc_update_partition_info_sn2(part, remote_rp_version,
|
|
|
&remote_rp_stamp, remote_rp_pa,
|
|
|
remote_vars_pa, remote_vars);
|
|
|
reactivate = 1;
|
|
|
-
|
|
|
- } else {
|
|
|
- DBUG_ON(!XPC_SUPPORTS_DISENGAGE_REQUEST(remote_vars->version));
|
|
|
-
|
|
|
- if (remote_rp_stamp != part->remote_rp_stamp) {
|
|
|
-
|
|
|
- /*
|
|
|
- * Other side rebooted and the previous XPC did support
|
|
|
- * the disengage request, as does the new one.
|
|
|
- */
|
|
|
-
|
|
|
- DBUG_ON(xpc_partition_engaged(1UL << partid));
|
|
|
- DBUG_ON(xpc_partition_disengage_requested(1UL <<
|
|
|
- partid));
|
|
|
-
|
|
|
- xpc_update_partition_info_sn2(part, remote_rp_version,
|
|
|
- &remote_rp_stamp,
|
|
|
- remote_rp_pa,
|
|
|
- remote_vars_pa,
|
|
|
- remote_vars);
|
|
|
- reactivate = 1;
|
|
|
- }
|
|
|
}
|
|
|
|
|
|
- if (part->disengage_request_timeout > 0 &&
|
|
|
- !xpc_partition_disengaged(part)) {
|
|
|
+ if (part->disengage_timeout > 0 && !xpc_partition_disengaged(part)) {
|
|
|
/* still waiting on other side to disengage from us */
|
|
|
return;
|
|
|
}
|
|
|
|
|
|
- if (reactivate) {
|
|
|
- part->reactivate_nasid = nasid;
|
|
|
+ if (reactivate)
|
|
|
XPC_DEACTIVATE_PARTITION(part, xpReactivating);
|
|
|
-
|
|
|
- } else if (XPC_SUPPORTS_DISENGAGE_REQUEST(part->remote_vars_version) &&
|
|
|
- xpc_partition_disengage_requested(1UL << partid)) {
|
|
|
+ else if (xpc_partition_deactivation_requested_sn2(partid))
|
|
|
XPC_DEACTIVATE_PARTITION(part, xpOtherGoingDown);
|
|
|
- }
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -1038,6 +1048,7 @@ xpc_process_activate_IRQ_rcvd_sn2(int n_IRQs_expected)
|
|
|
static enum xp_retval
|
|
|
xpc_setup_infrastructure_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
enum xp_retval retval;
|
|
|
int ret;
|
|
|
int cpuid;
|
|
@@ -1060,28 +1071,29 @@ xpc_setup_infrastructure_sn2(struct xpc_partition *part)
|
|
|
|
|
|
/* allocate all the required GET/PUT values */
|
|
|
|
|
|
- part->local_GPs = xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE,
|
|
|
- GFP_KERNEL,
|
|
|
- &part->local_GPs_base);
|
|
|
- if (part->local_GPs == NULL) {
|
|
|
+ part_sn2->local_GPs = xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE,
|
|
|
+ GFP_KERNEL,
|
|
|
+ &part_sn2->
|
|
|
+ local_GPs_base);
|
|
|
+ if (part_sn2->local_GPs == NULL) {
|
|
|
dev_err(xpc_chan, "can't get memory for local get/put "
|
|
|
"values\n");
|
|
|
retval = xpNoMemory;
|
|
|
goto out_1;
|
|
|
}
|
|
|
|
|
|
- part->remote_GPs = xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE,
|
|
|
- GFP_KERNEL,
|
|
|
- &part->
|
|
|
- remote_GPs_base);
|
|
|
- if (part->remote_GPs == NULL) {
|
|
|
+ part_sn2->remote_GPs = xpc_kzalloc_cacheline_aligned(XPC_GP_SIZE,
|
|
|
+ GFP_KERNEL,
|
|
|
+ &part_sn2->
|
|
|
+ remote_GPs_base);
|
|
|
+ if (part_sn2->remote_GPs == NULL) {
|
|
|
dev_err(xpc_chan, "can't get memory for remote get/put "
|
|
|
"values\n");
|
|
|
retval = xpNoMemory;
|
|
|
goto out_2;
|
|
|
}
|
|
|
|
|
|
- part->remote_GPs_pa = 0;
|
|
|
+ part_sn2->remote_GPs_pa = 0;
|
|
|
|
|
|
/* allocate all the required open and close args */
|
|
|
|
|
@@ -1103,22 +1115,23 @@ xpc_setup_infrastructure_sn2(struct xpc_partition *part)
|
|
|
goto out_4;
|
|
|
}
|
|
|
|
|
|
- part->remote_openclose_args_pa = 0;
|
|
|
+ part_sn2->remote_openclose_args_pa = 0;
|
|
|
|
|
|
- part->local_IPI_amo_va = xpc_IPI_init_sn2(partid);
|
|
|
+ part_sn2->local_IPI_amo_va = xpc_IPI_init_sn2(partid);
|
|
|
part->local_IPI_amo = 0;
|
|
|
spin_lock_init(&part->IPI_lock);
|
|
|
|
|
|
- part->remote_IPI_nasid = 0;
|
|
|
- part->remote_IPI_phys_cpuid = 0;
|
|
|
- part->remote_IPI_amo_va = NULL;
|
|
|
+ part_sn2->remote_IPI_nasid = 0;
|
|
|
+ part_sn2->remote_IPI_phys_cpuid = 0;
|
|
|
+ part_sn2->remote_IPI_amo_va = NULL;
|
|
|
|
|
|
atomic_set(&part->channel_mgr_requests, 1);
|
|
|
init_waitqueue_head(&part->channel_mgr_wq);
|
|
|
|
|
|
- sprintf(part->IPI_owner, "xpc%02d", partid);
|
|
|
- ret = request_irq(SGI_XPC_NOTIFY, xpc_notify_IRQ_handler, IRQF_SHARED,
|
|
|
- part->IPI_owner, (void *)(u64)partid);
|
|
|
+ sprintf(part_sn2->IPI_owner, "xpc%02d", partid);
|
|
|
+ ret = request_irq(SGI_XPC_NOTIFY, xpc_handle_notify_IRQ_sn2,
|
|
|
+ IRQF_SHARED, part_sn2->IPI_owner,
|
|
|
+ (void *)(u64)partid);
|
|
|
if (ret != 0) {
|
|
|
dev_err(xpc_chan, "can't register NOTIFY IRQ handler, "
|
|
|
"errno=%d\n", -ret);
|
|
@@ -1127,9 +1140,10 @@ xpc_setup_infrastructure_sn2(struct xpc_partition *part)
|
|
|
}
|
|
|
|
|
|
/* Setup a timer to check for dropped IPIs */
|
|
|
- timer = &part->dropped_IPI_timer;
|
|
|
+ timer = &part_sn2->dropped_notify_IRQ_timer;
|
|
|
init_timer(timer);
|
|
|
- timer->function = (void (*)(unsigned long))xpc_dropped_IPI_check;
|
|
|
+ timer->function =
|
|
|
+ (void (*)(unsigned long))xpc_dropped_notify_IRQ_check_sn2;
|
|
|
timer->data = (unsigned long)part;
|
|
|
timer->expires = jiffies + XPC_P_DROPPED_IPI_WAIT_INTERVAL;
|
|
|
add_timer(timer);
|
|
@@ -1146,7 +1160,7 @@ xpc_setup_infrastructure_sn2(struct xpc_partition *part)
|
|
|
ch->number = ch_number;
|
|
|
ch->flags = XPC_C_DISCONNECTED;
|
|
|
|
|
|
- ch->local_GP = &part->local_GPs[ch_number];
|
|
|
+ ch->sn.sn2.local_GP = &part_sn2->local_GPs[ch_number];
|
|
|
ch->local_openclose_args =
|
|
|
&part->local_openclose_args[ch_number];
|
|
|
|
|
@@ -1158,7 +1172,7 @@ xpc_setup_infrastructure_sn2(struct xpc_partition *part)
|
|
|
atomic_set(&ch->n_to_notify, 0);
|
|
|
|
|
|
spin_lock_init(&ch->lock);
|
|
|
- mutex_init(&ch->msg_to_pull_mutex);
|
|
|
+ mutex_init(&ch->sn.sn2.msg_to_pull_mutex);
|
|
|
init_completion(&ch->wdisconnect_wait);
|
|
|
|
|
|
atomic_set(&ch->n_on_msg_allocate_wq, 0);
|
|
@@ -1179,10 +1193,10 @@ xpc_setup_infrastructure_sn2(struct xpc_partition *part)
|
|
|
* The setting of the magic # indicates that these per partition
|
|
|
* specific variables are ready to be used.
|
|
|
*/
|
|
|
- xpc_vars_part[partid].GPs_pa = __pa(part->local_GPs);
|
|
|
+ xpc_vars_part[partid].GPs_pa = __pa(part_sn2->local_GPs);
|
|
|
xpc_vars_part[partid].openclose_args_pa =
|
|
|
__pa(part->local_openclose_args);
|
|
|
- xpc_vars_part[partid].IPI_amo_pa = __pa(part->local_IPI_amo_va);
|
|
|
+ xpc_vars_part[partid].IPI_amo_pa = __pa(part_sn2->local_IPI_amo_va);
|
|
|
cpuid = raw_smp_processor_id(); /* any CPU in this partition will do */
|
|
|
xpc_vars_part[partid].IPI_nasid = cpuid_to_nasid(cpuid);
|
|
|
xpc_vars_part[partid].IPI_phys_cpuid = cpu_physical_id(cpuid);
|
|
@@ -1199,11 +1213,11 @@ out_4:
|
|
|
kfree(part->local_openclose_args_base);
|
|
|
part->local_openclose_args = NULL;
|
|
|
out_3:
|
|
|
- kfree(part->remote_GPs_base);
|
|
|
- part->remote_GPs = NULL;
|
|
|
+ kfree(part_sn2->remote_GPs_base);
|
|
|
+ part_sn2->remote_GPs = NULL;
|
|
|
out_2:
|
|
|
- kfree(part->local_GPs_base);
|
|
|
- part->local_GPs = NULL;
|
|
|
+ kfree(part_sn2->local_GPs_base);
|
|
|
+ part_sn2->local_GPs = NULL;
|
|
|
out_1:
|
|
|
kfree(part->channels);
|
|
|
part->channels = NULL;
|
|
@@ -1217,6 +1231,7 @@ out_1:
|
|
|
static void
|
|
|
xpc_teardown_infrastructure_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
short partid = XPC_PARTID(part);
|
|
|
|
|
|
/*
|
|
@@ -1248,19 +1263,19 @@ xpc_teardown_infrastructure_sn2(struct xpc_partition *part)
|
|
|
part->setup_state = XPC_P_TORNDOWN;
|
|
|
|
|
|
/* in case we've still got outstanding timers registered... */
|
|
|
- del_timer_sync(&part->dropped_IPI_timer);
|
|
|
+ del_timer_sync(&part_sn2->dropped_notify_IRQ_timer);
|
|
|
|
|
|
kfree(part->remote_openclose_args_base);
|
|
|
part->remote_openclose_args = NULL;
|
|
|
kfree(part->local_openclose_args_base);
|
|
|
part->local_openclose_args = NULL;
|
|
|
- kfree(part->remote_GPs_base);
|
|
|
- part->remote_GPs = NULL;
|
|
|
- kfree(part->local_GPs_base);
|
|
|
- part->local_GPs = NULL;
|
|
|
+ kfree(part_sn2->remote_GPs_base);
|
|
|
+ part_sn2->remote_GPs = NULL;
|
|
|
+ kfree(part_sn2->local_GPs_base);
|
|
|
+ part_sn2->local_GPs = NULL;
|
|
|
kfree(part->channels);
|
|
|
part->channels = NULL;
|
|
|
- part->local_IPI_amo_va = NULL;
|
|
|
+ part_sn2->local_IPI_amo_va = NULL;
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -1300,6 +1315,7 @@ xpc_pull_remote_cachelines_sn2(struct xpc_partition *part, void *dst,
|
|
|
static enum xp_retval
|
|
|
xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
u8 buffer[L1_CACHE_BYTES * 2];
|
|
|
struct xpc_vars_part_sn2 *pulled_entry_cacheline =
|
|
|
(struct xpc_vars_part_sn2 *)L1_CACHE_ALIGN((u64)buffer);
|
|
@@ -1310,11 +1326,11 @@ xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
|
|
|
|
|
|
/* pull the cacheline that contains the variables we're interested in */
|
|
|
|
|
|
- DBUG_ON(part->remote_vars_part_pa !=
|
|
|
- L1_CACHE_ALIGN(part->remote_vars_part_pa));
|
|
|
+ DBUG_ON(part_sn2->remote_vars_part_pa !=
|
|
|
+ L1_CACHE_ALIGN(part_sn2->remote_vars_part_pa));
|
|
|
DBUG_ON(sizeof(struct xpc_vars_part_sn2) != L1_CACHE_BYTES / 2);
|
|
|
|
|
|
- remote_entry_pa = part->remote_vars_part_pa +
|
|
|
+ remote_entry_pa = part_sn2->remote_vars_part_pa +
|
|
|
sn_partition_id * sizeof(struct xpc_vars_part_sn2);
|
|
|
|
|
|
remote_entry_cacheline_pa = (remote_entry_pa & ~(L1_CACHE_BYTES - 1));
|
|
@@ -1364,13 +1380,13 @@ xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
|
|
|
|
|
|
/* the variables we imported look to be valid */
|
|
|
|
|
|
- part->remote_GPs_pa = pulled_entry->GPs_pa;
|
|
|
- part->remote_openclose_args_pa =
|
|
|
+ part_sn2->remote_GPs_pa = pulled_entry->GPs_pa;
|
|
|
+ part_sn2->remote_openclose_args_pa =
|
|
|
pulled_entry->openclose_args_pa;
|
|
|
- part->remote_IPI_amo_va =
|
|
|
+ part_sn2->remote_IPI_amo_va =
|
|
|
(AMO_t *)__va(pulled_entry->IPI_amo_pa);
|
|
|
- part->remote_IPI_nasid = pulled_entry->IPI_nasid;
|
|
|
- part->remote_IPI_phys_cpuid = pulled_entry->IPI_phys_cpuid;
|
|
|
+ part_sn2->remote_IPI_nasid = pulled_entry->IPI_nasid;
|
|
|
+ part_sn2->remote_IPI_phys_cpuid = pulled_entry->IPI_phys_cpuid;
|
|
|
|
|
|
if (part->nchannels > pulled_entry->nchannels)
|
|
|
part->nchannels = pulled_entry->nchannels;
|
|
@@ -1394,6 +1410,7 @@ xpc_pull_remote_vars_part_sn2(struct xpc_partition *part)
|
|
|
static enum xp_retval
|
|
|
xpc_make_first_contact_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
enum xp_retval ret;
|
|
|
|
|
|
/*
|
|
@@ -1406,7 +1423,7 @@ xpc_make_first_contact_sn2(struct xpc_partition *part)
|
|
|
* we should get the same page for remote_amos_page_pa after module
|
|
|
* reloads and system reboots.
|
|
|
*/
|
|
|
- if (sn_register_xp_addr_region(part->remote_amos_page_pa,
|
|
|
+ if (sn_register_xp_addr_region(part_sn2->remote_amos_page_pa,
|
|
|
PAGE_SIZE, 1) < 0) {
|
|
|
dev_warn(xpc_part, "xpc_activating(%d) failed to register "
|
|
|
"xp_addr region\n", XPC_PARTID(part));
|
|
@@ -1416,7 +1433,14 @@ xpc_make_first_contact_sn2(struct xpc_partition *part)
|
|
|
return ret;
|
|
|
}
|
|
|
|
|
|
- xpc_IPI_send_activated(part);
|
|
|
+ /*
|
|
|
+ * Send activate IRQ to get other side to activate if they've not
|
|
|
+ * already begun to do so.
|
|
|
+ */
|
|
|
+ xpc_activate_IRQ_send_sn2(part_sn2->remote_amos_page_pa,
|
|
|
+ cnodeid_to_nasid(0),
|
|
|
+ part_sn2->activate_IRQ_nasid,
|
|
|
+ part_sn2->activate_IRQ_phys_cpuid);
|
|
|
|
|
|
while ((ret = xpc_pull_remote_vars_part_sn2(part)) != xpSuccess) {
|
|
|
if (ret != xpRetry) {
|
|
@@ -1443,6 +1467,7 @@ xpc_make_first_contact_sn2(struct xpc_partition *part)
|
|
|
static u64
|
|
|
xpc_get_IPI_flags_sn2(struct xpc_partition *part)
|
|
|
{
|
|
|
+ struct xpc_partition_sn2 *part_sn2 = &part->sn.sn2;
|
|
|
unsigned long irq_flags;
|
|
|
u64 IPI_amo;
|
|
|
enum xp_retval ret;
|
|
@@ -1459,9 +1484,9 @@ xpc_get_IPI_flags_sn2(struct xpc_partition *part)
|
|
|
spin_unlock_irqrestore(&part->IPI_lock, irq_flags);
|
|
|
|
|
|
if (XPC_ANY_OPENCLOSE_IPI_FLAGS_SET(IPI_amo)) {
|
|
|
- ret = xpc_pull_remote_cachelines_sn2(part,
|
|
|
- part->remote_openclose_args,
|
|
|
- (void *)part->
|
|
|
+ ret = xpc_pull_remote_cachelines_sn2(part, part->
|
|
|
+ remote_openclose_args,
|
|
|
+ (void *)part_sn2->
|
|
|
remote_openclose_args_pa,
|
|
|
XPC_OPENCLOSE_ARGS_SIZE);
|
|
|
if (ret != xpSuccess) {
|
|
@@ -1477,8 +1502,8 @@ xpc_get_IPI_flags_sn2(struct xpc_partition *part)
|
|
|
}
|
|
|
|
|
|
if (XPC_ANY_MSG_IPI_FLAGS_SET(IPI_amo)) {
|
|
|
- ret = xpc_pull_remote_cachelines_sn2(part, part->remote_GPs,
|
|
|
- (void *)part->remote_GPs_pa,
|
|
|
+ ret = xpc_pull_remote_cachelines_sn2(part, part_sn2->remote_GPs,
|
|
|
+ (void *)part_sn2->remote_GPs_pa,
|
|
|
XPC_GP_SIZE);
|
|
|
if (ret != xpSuccess) {
|
|
|
XPC_DEACTIVATE_PARTITION(part, ret);
|
|
@@ -1494,28 +1519,220 @@ xpc_get_IPI_flags_sn2(struct xpc_partition *part)
|
|
|
return IPI_amo;
|
|
|
}
|
|
|
|
|
|
+/*
|
|
|
+ * Notify those who wanted to be notified upon delivery of their message.
|
|
|
+ */
|
|
|
+static void
|
|
|
+xpc_notify_senders_sn2(struct xpc_channel *ch, enum xp_retval reason, s64 put)
|
|
|
+{
|
|
|
+ struct xpc_notify *notify;
|
|
|
+ u8 notify_type;
|
|
|
+ s64 get = ch->sn.sn2.w_remote_GP.get - 1;
|
|
|
+
|
|
|
+ while (++get < put && atomic_read(&ch->n_to_notify) > 0) {
|
|
|
+
|
|
|
+ notify = &ch->notify_queue[get % ch->local_nentries];
|
|
|
+
|
|
|
+ /*
|
|
|
+ * See if the notify entry indicates it was associated with
|
|
|
+ * a message who's sender wants to be notified. It is possible
|
|
|
+ * that it is, but someone else is doing or has done the
|
|
|
+ * notification.
|
|
|
+ */
|
|
|
+ notify_type = notify->type;
|
|
|
+ if (notify_type == 0 ||
|
|
|
+ cmpxchg(¬ify->type, notify_type, 0) != notify_type) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+
|
|
|
+ DBUG_ON(notify_type != XPC_N_CALL);
|
|
|
+
|
|
|
+ atomic_dec(&ch->n_to_notify);
|
|
|
+
|
|
|
+ if (notify->func != NULL) {
|
|
|
+ dev_dbg(xpc_chan, "notify->func() called, notify=0x%p, "
|
|
|
+ "msg_number=%ld, partid=%d, channel=%d\n",
|
|
|
+ (void *)notify, get, ch->partid, ch->number);
|
|
|
+
|
|
|
+ notify->func(reason, ch->partid, ch->number,
|
|
|
+ notify->key);
|
|
|
+
|
|
|
+ dev_dbg(xpc_chan, "notify->func() returned, "
|
|
|
+ "notify=0x%p, msg_number=%ld, partid=%d, "
|
|
|
+ "channel=%d\n", (void *)notify, get,
|
|
|
+ ch->partid, ch->number);
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+static void
|
|
|
+xpc_notify_senders_of_disconnect_sn2(struct xpc_channel *ch)
|
|
|
+{
|
|
|
+ xpc_notify_senders_sn2(ch, ch->reason, ch->sn.sn2.w_local_GP.put);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Clear some of the msg flags in the local message queue.
|
|
|
+ */
|
|
|
+static inline void
|
|
|
+xpc_clear_local_msgqueue_flags_sn2(struct xpc_channel *ch)
|
|
|
+{
|
|
|
+ struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
|
|
|
+ struct xpc_msg *msg;
|
|
|
+ s64 get;
|
|
|
+
|
|
|
+ get = ch_sn2->w_remote_GP.get;
|
|
|
+ do {
|
|
|
+ msg = (struct xpc_msg *)((u64)ch->local_msgqueue +
|
|
|
+ (get % ch->local_nentries) *
|
|
|
+ ch->msg_size);
|
|
|
+ msg->flags = 0;
|
|
|
+ } while (++get < ch_sn2->remote_GP.get);
|
|
|
+}
|
|
|
+
|
|
|
+/*
|
|
|
+ * Clear some of the msg flags in the remote message queue.
|
|
|
+ */
|
|
|
+static inline void
|
|
|
+xpc_clear_remote_msgqueue_flags_sn2(struct xpc_channel *ch)
|
|
|
+{
|
|
|
+ struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
|
|
|
+ struct xpc_msg *msg;
|
|
|
+ s64 put;
|
|
|
+
|
|
|
+ put = ch_sn2->w_remote_GP.put;
|
|
|
+ do {
|
|
|
+ msg = (struct xpc_msg *)((u64)ch->remote_msgqueue +
|
|
|
+ (put % ch->remote_nentries) *
|
|
|
+ ch->msg_size);
|
|
|
+ msg->flags = 0;
|
|
|
+ } while (++put < ch_sn2->remote_GP.put);
|
|
|
+}
|
|
|
+
|
|
|
+static void
|
|
|
+xpc_process_msg_IPI_sn2(struct xpc_partition *part, int ch_number)
|
|
|
+{
|
|
|
+ struct xpc_channel *ch = &part->channels[ch_number];
|
|
|
+ struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
|
|
|
+ int nmsgs_sent;
|
|
|
+
|
|
|
+ ch_sn2->remote_GP = part->sn.sn2.remote_GPs[ch_number];
|
|
|
+
|
|
|
+ /* See what, if anything, has changed for each connected channel */
|
|
|
+
|
|
|
+ xpc_msgqueue_ref(ch);
|
|
|
+
|
|
|
+ if (ch_sn2->w_remote_GP.get == ch_sn2->remote_GP.get &&
|
|
|
+ ch_sn2->w_remote_GP.put == ch_sn2->remote_GP.put) {
|
|
|
+ /* nothing changed since GPs were last pulled */
|
|
|
+ xpc_msgqueue_deref(ch);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ if (!(ch->flags & XPC_C_CONNECTED)) {
|
|
|
+ xpc_msgqueue_deref(ch);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * First check to see if messages recently sent by us have been
|
|
|
+ * received by the other side. (The remote GET value will have
|
|
|
+ * changed since we last looked at it.)
|
|
|
+ */
|
|
|
+
|
|
|
+ if (ch_sn2->w_remote_GP.get != ch_sn2->remote_GP.get) {
|
|
|
+
|
|
|
+ /*
|
|
|
+ * We need to notify any senders that want to be notified
|
|
|
+ * that their sent messages have been received by their
|
|
|
+ * intended recipients. We need to do this before updating
|
|
|
+ * w_remote_GP.get so that we don't allocate the same message
|
|
|
+ * queue entries prematurely (see xpc_allocate_msg()).
|
|
|
+ */
|
|
|
+ if (atomic_read(&ch->n_to_notify) > 0) {
|
|
|
+ /*
|
|
|
+ * Notify senders that messages sent have been
|
|
|
+ * received and delivered by the other side.
|
|
|
+ */
|
|
|
+ xpc_notify_senders_sn2(ch, xpMsgDelivered,
|
|
|
+ ch_sn2->remote_GP.get);
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Clear msg->flags in previously sent messages, so that
|
|
|
+ * they're ready for xpc_allocate_msg().
|
|
|
+ */
|
|
|
+ xpc_clear_local_msgqueue_flags_sn2(ch);
|
|
|
+
|
|
|
+ ch_sn2->w_remote_GP.get = ch_sn2->remote_GP.get;
|
|
|
+
|
|
|
+ dev_dbg(xpc_chan, "w_remote_GP.get changed to %ld, partid=%d, "
|
|
|
+ "channel=%d\n", ch_sn2->w_remote_GP.get, ch->partid,
|
|
|
+ ch->number);
|
|
|
+
|
|
|
+ /*
|
|
|
+ * If anyone was waiting for message queue entries to become
|
|
|
+ * available, wake them up.
|
|
|
+ */
|
|
|
+ if (atomic_read(&ch->n_on_msg_allocate_wq) > 0)
|
|
|
+ wake_up(&ch->msg_allocate_wq);
|
|
|
+ }
|
|
|
+
|
|
|
+ /*
|
|
|
+ * Now check for newly sent messages by the other side. (The remote
|
|
|
+ * PUT value will have changed since we last looked at it.)
|
|
|
+ */
|
|
|
+
|
|
|
+ if (ch_sn2->w_remote_GP.put != ch_sn2->remote_GP.put) {
|
|
|
+ /*
|
|
|
+ * Clear msg->flags in previously received messages, so that
|
|
|
+ * they're ready for xpc_get_deliverable_msg().
|
|
|
+ */
|
|
|
+ xpc_clear_remote_msgqueue_flags_sn2(ch);
|
|
|
+
|
|
|
+ ch_sn2->w_remote_GP.put = ch_sn2->remote_GP.put;
|
|
|
+
|
|
|
+ dev_dbg(xpc_chan, "w_remote_GP.put changed to %ld, partid=%d, "
|
|
|
+ "channel=%d\n", ch_sn2->w_remote_GP.put, ch->partid,
|
|
|
+ ch->number);
|
|
|
+
|
|
|
+ nmsgs_sent = ch_sn2->w_remote_GP.put - ch_sn2->w_local_GP.get;
|
|
|
+ if (nmsgs_sent > 0) {
|
|
|
+ dev_dbg(xpc_chan, "msgs waiting to be copied and "
|
|
|
+ "delivered=%d, partid=%d, channel=%d\n",
|
|
|
+ nmsgs_sent, ch->partid, ch->number);
|
|
|
+
|
|
|
+ if (ch->flags & XPC_C_CONNECTEDCALLOUT_MADE)
|
|
|
+ xpc_activate_kthreads(ch, nmsgs_sent);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ xpc_msgqueue_deref(ch);
|
|
|
+}
|
|
|
+
|
|
|
static struct xpc_msg *
|
|
|
xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
|
|
|
{
|
|
|
struct xpc_partition *part = &xpc_partitions[ch->partid];
|
|
|
+ struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
|
|
|
struct xpc_msg *remote_msg, *msg;
|
|
|
u32 msg_index, nmsgs;
|
|
|
u64 msg_offset;
|
|
|
enum xp_retval ret;
|
|
|
|
|
|
- if (mutex_lock_interruptible(&ch->msg_to_pull_mutex) != 0) {
|
|
|
+ if (mutex_lock_interruptible(&ch_sn2->msg_to_pull_mutex) != 0) {
|
|
|
/* we were interrupted by a signal */
|
|
|
return NULL;
|
|
|
}
|
|
|
|
|
|
- while (get >= ch->next_msg_to_pull) {
|
|
|
+ while (get >= ch_sn2->next_msg_to_pull) {
|
|
|
|
|
|
/* pull as many messages as are ready and able to be pulled */
|
|
|
|
|
|
- msg_index = ch->next_msg_to_pull % ch->remote_nentries;
|
|
|
+ msg_index = ch_sn2->next_msg_to_pull % ch->remote_nentries;
|
|
|
|
|
|
- DBUG_ON(ch->next_msg_to_pull >= ch->w_remote_GP.put);
|
|
|
- nmsgs = ch->w_remote_GP.put - ch->next_msg_to_pull;
|
|
|
+ DBUG_ON(ch_sn2->next_msg_to_pull >= ch_sn2->w_remote_GP.put);
|
|
|
+ nmsgs = ch_sn2->w_remote_GP.put - ch_sn2->next_msg_to_pull;
|
|
|
if (msg_index + nmsgs > ch->remote_nentries) {
|
|
|
/* ignore the ones that wrap the msg queue for now */
|
|
|
nmsgs = ch->remote_nentries - msg_index;
|
|
@@ -1532,19 +1749,19 @@ xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
|
|
|
|
|
|
dev_dbg(xpc_chan, "failed to pull %d msgs starting with"
|
|
|
" msg %ld from partition %d, channel=%d, "
|
|
|
- "ret=%d\n", nmsgs, ch->next_msg_to_pull,
|
|
|
+ "ret=%d\n", nmsgs, ch_sn2->next_msg_to_pull,
|
|
|
ch->partid, ch->number, ret);
|
|
|
|
|
|
XPC_DEACTIVATE_PARTITION(part, ret);
|
|
|
|
|
|
- mutex_unlock(&ch->msg_to_pull_mutex);
|
|
|
+ mutex_unlock(&ch_sn2->msg_to_pull_mutex);
|
|
|
return NULL;
|
|
|
}
|
|
|
|
|
|
- ch->next_msg_to_pull += nmsgs;
|
|
|
+ ch_sn2->next_msg_to_pull += nmsgs;
|
|
|
}
|
|
|
|
|
|
- mutex_unlock(&ch->msg_to_pull_mutex);
|
|
|
+ mutex_unlock(&ch_sn2->msg_to_pull_mutex);
|
|
|
|
|
|
/* return the message we were looking for */
|
|
|
msg_offset = (get % ch->remote_nentries) * ch->msg_size;
|
|
@@ -1553,12 +1770,19 @@ xpc_pull_remote_msg_sn2(struct xpc_channel *ch, s64 get)
|
|
|
return msg;
|
|
|
}
|
|
|
|
|
|
+static int
|
|
|
+xpc_n_of_deliverable_msgs_sn2(struct xpc_channel *ch)
|
|
|
+{
|
|
|
+ return ch->sn.sn2.w_remote_GP.put - ch->sn.sn2.w_local_GP.get;
|
|
|
+}
|
|
|
+
|
|
|
/*
|
|
|
* Get a message to be delivered.
|
|
|
*/
|
|
|
static struct xpc_msg *
|
|
|
xpc_get_deliverable_msg_sn2(struct xpc_channel *ch)
|
|
|
{
|
|
|
+ struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
|
|
|
struct xpc_msg *msg = NULL;
|
|
|
s64 get;
|
|
|
|
|
@@ -1566,9 +1790,9 @@ xpc_get_deliverable_msg_sn2(struct xpc_channel *ch)
|
|
|
if (ch->flags & XPC_C_DISCONNECTING)
|
|
|
break;
|
|
|
|
|
|
- get = ch->w_local_GP.get;
|
|
|
+ get = ch_sn2->w_local_GP.get;
|
|
|
rmb(); /* guarantee that .get loads before .put */
|
|
|
- if (get == ch->w_remote_GP.put)
|
|
|
+ if (get == ch_sn2->w_remote_GP.put)
|
|
|
break;
|
|
|
|
|
|
/* There are messages waiting to be pulled and delivered.
|
|
@@ -1578,7 +1802,7 @@ xpc_get_deliverable_msg_sn2(struct xpc_channel *ch)
|
|
|
* to try again for the next one.
|
|
|
*/
|
|
|
|
|
|
- if (cmpxchg(&ch->w_local_GP.get, get, get + 1) == get) {
|
|
|
+ if (cmpxchg(&ch_sn2->w_local_GP.get, get, get + 1) == get) {
|
|
|
/* we got the entry referenced by get */
|
|
|
|
|
|
dev_dbg(xpc_chan, "w_local_GP.get changed to %ld, "
|
|
@@ -1609,6 +1833,7 @@ xpc_get_deliverable_msg_sn2(struct xpc_channel *ch)
|
|
|
static void
|
|
|
xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
|
|
|
{
|
|
|
+ struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
|
|
|
struct xpc_msg *msg;
|
|
|
s64 put = initial_put + 1;
|
|
|
int send_IPI = 0;
|
|
@@ -1616,7 +1841,7 @@ xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
|
|
|
while (1) {
|
|
|
|
|
|
while (1) {
|
|
|
- if (put == ch->w_local_GP.put)
|
|
|
+ if (put == ch_sn2->w_local_GP.put)
|
|
|
break;
|
|
|
|
|
|
msg = (struct xpc_msg *)((u64)ch->local_msgqueue +
|
|
@@ -1634,10 +1859,10 @@ xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
|
|
|
break;
|
|
|
}
|
|
|
|
|
|
- if (cmpxchg_rel(&ch->local_GP->put, initial_put, put) !=
|
|
|
+ if (cmpxchg_rel(&ch_sn2->local_GP->put, initial_put, put) !=
|
|
|
initial_put) {
|
|
|
/* someone else beat us to it */
|
|
|
- DBUG_ON(ch->local_GP->put < initial_put);
|
|
|
+ DBUG_ON(ch_sn2->local_GP->put < initial_put);
|
|
|
break;
|
|
|
}
|
|
|
|
|
@@ -1657,7 +1882,7 @@ xpc_send_msgs_sn2(struct xpc_channel *ch, s64 initial_put)
|
|
|
}
|
|
|
|
|
|
if (send_IPI)
|
|
|
- xpc_IPI_send_msgrequest_sn2(ch);
|
|
|
+ xpc_send_channel_msgrequest_sn2(ch);
|
|
|
}
|
|
|
|
|
|
/*
|
|
@@ -1668,6 +1893,7 @@ static enum xp_retval
|
|
|
xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
|
|
|
struct xpc_msg **address_of_msg)
|
|
|
{
|
|
|
+ struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
|
|
|
struct xpc_msg *msg;
|
|
|
enum xp_retval ret;
|
|
|
s64 put;
|
|
@@ -1681,9 +1907,9 @@ xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
|
|
|
|
|
|
while (1) {
|
|
|
|
|
|
- put = ch->w_local_GP.put;
|
|
|
+ put = ch_sn2->w_local_GP.put;
|
|
|
rmb(); /* guarantee that .put loads before .get */
|
|
|
- if (put - ch->w_remote_GP.get < ch->local_nentries) {
|
|
|
+ if (put - ch_sn2->w_remote_GP.get < ch->local_nentries) {
|
|
|
|
|
|
/* There are available message entries. We need to try
|
|
|
* to secure one for ourselves. We'll do this by trying
|
|
@@ -1691,7 +1917,8 @@ xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
|
|
|
* doesn't beat us to it. If they do, we'll have to
|
|
|
* try again.
|
|
|
*/
|
|
|
- if (cmpxchg(&ch->w_local_GP.put, put, put + 1) == put) {
|
|
|
+ if (cmpxchg(&ch_sn2->w_local_GP.put, put, put + 1) ==
|
|
|
+ put) {
|
|
|
/* we got the entry referenced by put */
|
|
|
break;
|
|
|
}
|
|
@@ -1708,7 +1935,7 @@ xpc_allocate_msg_sn2(struct xpc_channel *ch, u32 flags,
|
|
|
* GP values as if an IPI was sent by the other side.
|
|
|
*/
|
|
|
if (ret == xpTimeout)
|
|
|
- xpc_IPI_send_local_msgrequest_sn2(ch);
|
|
|
+ xpc_send_channel_local_msgrequest_sn2(ch);
|
|
|
|
|
|
if (flags & XPC_NOWAIT)
|
|
|
return xpNoWait;
|
|
@@ -1810,13 +2037,13 @@ xpc_send_msg_sn2(struct xpc_channel *ch, u32 flags, void *payload,
|
|
|
|
|
|
/*
|
|
|
* The preceding store of msg->flags must occur before the following
|
|
|
- * load of ch->local_GP->put.
|
|
|
+ * load of local_GP->put.
|
|
|
*/
|
|
|
mb();
|
|
|
|
|
|
/* see if the message is next in line to be sent, if so send it */
|
|
|
|
|
|
- put = ch->local_GP->put;
|
|
|
+ put = ch->sn.sn2.local_GP->put;
|
|
|
if (put == msg_number)
|
|
|
xpc_send_msgs_sn2(ch, put);
|
|
|
|
|
@@ -1833,6 +2060,7 @@ out_1:
|
|
|
static void
|
|
|
xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
|
|
|
{
|
|
|
+ struct xpc_channel_sn2 *ch_sn2 = &ch->sn.sn2;
|
|
|
struct xpc_msg *msg;
|
|
|
s64 get = initial_get + 1;
|
|
|
int send_IPI = 0;
|
|
@@ -1840,7 +2068,7 @@ xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
|
|
|
while (1) {
|
|
|
|
|
|
while (1) {
|
|
|
- if (get == ch->w_local_GP.get)
|
|
|
+ if (get == ch_sn2->w_local_GP.get)
|
|
|
break;
|
|
|
|
|
|
msg = (struct xpc_msg *)((u64)ch->remote_msgqueue +
|
|
@@ -1859,10 +2087,10 @@ xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
|
|
|
break;
|
|
|
}
|
|
|
|
|
|
- if (cmpxchg_rel(&ch->local_GP->get, initial_get, get) !=
|
|
|
+ if (cmpxchg_rel(&ch_sn2->local_GP->get, initial_get, get) !=
|
|
|
initial_get) {
|
|
|
/* someone else beat us to it */
|
|
|
- DBUG_ON(ch->local_GP->get <= initial_get);
|
|
|
+ DBUG_ON(ch_sn2->local_GP->get <= initial_get);
|
|
|
break;
|
|
|
}
|
|
|
|
|
@@ -1882,7 +2110,7 @@ xpc_acknowledge_msgs_sn2(struct xpc_channel *ch, s64 initial_get, u8 msg_flags)
|
|
|
}
|
|
|
|
|
|
if (send_IPI)
|
|
|
- xpc_IPI_send_msgrequest_sn2(ch);
|
|
|
+ xpc_send_channel_msgrequest_sn2(ch);
|
|
|
}
|
|
|
|
|
|
static void
|
|
@@ -1902,7 +2130,7 @@ xpc_received_msg_sn2(struct xpc_channel *ch, struct xpc_msg *msg)
|
|
|
|
|
|
/*
|
|
|
* The preceding store of msg->flags must occur before the following
|
|
|
- * load of ch->local_GP->get.
|
|
|
+ * load of local_GP->get.
|
|
|
*/
|
|
|
mb();
|
|
|
|
|
@@ -1910,7 +2138,7 @@ xpc_received_msg_sn2(struct xpc_channel *ch, struct xpc_msg *msg)
|
|
|
* See if this message is next in line to be acknowledged as having
|
|
|
* been delivered.
|
|
|
*/
|
|
|
- get = ch->local_GP->get;
|
|
|
+ get = ch->sn.sn2.local_GP->get;
|
|
|
if (get == msg_number)
|
|
|
xpc_acknowledge_msgs_sn2(ch, get, msg->flags);
|
|
|
}
|
|
@@ -1928,36 +2156,35 @@ xpc_init_sn2(void)
|
|
|
xpc_heartbeat_exit = xpc_heartbeat_exit_sn2;
|
|
|
xpc_check_remote_hb = xpc_check_remote_hb_sn2;
|
|
|
|
|
|
- xpc_initiate_partition_activation =
|
|
|
- xpc_initiate_partition_activation_sn2;
|
|
|
+ xpc_request_partition_activation = xpc_request_partition_activation_sn2;
|
|
|
+ xpc_request_partition_reactivation =
|
|
|
+ xpc_request_partition_reactivation_sn2;
|
|
|
+ xpc_request_partition_deactivation =
|
|
|
+ xpc_request_partition_deactivation_sn2;
|
|
|
+ xpc_cancel_partition_deactivation_request =
|
|
|
+ xpc_cancel_partition_deactivation_request_sn2;
|
|
|
+
|
|
|
xpc_process_activate_IRQ_rcvd = xpc_process_activate_IRQ_rcvd_sn2;
|
|
|
xpc_setup_infrastructure = xpc_setup_infrastructure_sn2;
|
|
|
xpc_teardown_infrastructure = xpc_teardown_infrastructure_sn2;
|
|
|
xpc_make_first_contact = xpc_make_first_contact_sn2;
|
|
|
xpc_get_IPI_flags = xpc_get_IPI_flags_sn2;
|
|
|
+ xpc_notify_senders_of_disconnect = xpc_notify_senders_of_disconnect_sn2;
|
|
|
+ xpc_process_msg_IPI = xpc_process_msg_IPI_sn2;
|
|
|
+ xpc_n_of_deliverable_msgs = xpc_n_of_deliverable_msgs_sn2;
|
|
|
xpc_get_deliverable_msg = xpc_get_deliverable_msg_sn2;
|
|
|
|
|
|
- xpc_mark_partition_engaged = xpc_mark_partition_engaged_sn2;
|
|
|
- xpc_mark_partition_disengaged = xpc_mark_partition_disengaged_sn2;
|
|
|
- xpc_request_partition_disengage = xpc_request_partition_disengage_sn2;
|
|
|
- xpc_cancel_partition_disengage_request =
|
|
|
- xpc_cancel_partition_disengage_request_sn2;
|
|
|
+ xpc_indicate_partition_engaged = xpc_indicate_partition_engaged_sn2;
|
|
|
xpc_partition_engaged = xpc_partition_engaged_sn2;
|
|
|
- xpc_partition_disengage_requested =
|
|
|
- xpc_partition_disengage_requested_sn2;
|
|
|
- xpc_clear_partition_engaged = xpc_clear_partition_engaged_sn2;
|
|
|
- xpc_clear_partition_disengage_request =
|
|
|
- xpc_clear_partition_disengage_request_sn2;
|
|
|
-
|
|
|
- xpc_IPI_send_local_activate = xpc_IPI_send_local_activate_sn2;
|
|
|
- xpc_IPI_send_activated = xpc_IPI_send_activated_sn2;
|
|
|
- xpc_IPI_send_local_reactivate = xpc_IPI_send_local_reactivate_sn2;
|
|
|
- xpc_IPI_send_disengage = xpc_IPI_send_disengage_sn2;
|
|
|
-
|
|
|
- xpc_IPI_send_closerequest = xpc_IPI_send_closerequest_sn2;
|
|
|
- xpc_IPI_send_closereply = xpc_IPI_send_closereply_sn2;
|
|
|
- xpc_IPI_send_openrequest = xpc_IPI_send_openrequest_sn2;
|
|
|
- xpc_IPI_send_openreply = xpc_IPI_send_openreply_sn2;
|
|
|
+ xpc_any_partition_engaged = xpc_any_partition_engaged_sn2;
|
|
|
+ xpc_indicate_partition_disengaged =
|
|
|
+ xpc_indicate_partition_disengaged_sn2;
|
|
|
+ xpc_assume_partition_disengaged = xpc_assume_partition_disengaged_sn2;
|
|
|
+
|
|
|
+ xpc_send_channel_closerequest = xpc_send_channel_closerequest_sn2;
|
|
|
+ xpc_send_channel_closereply = xpc_send_channel_closereply_sn2;
|
|
|
+ xpc_send_channel_openrequest = xpc_send_channel_openrequest_sn2;
|
|
|
+ xpc_send_channel_openreply = xpc_send_channel_openreply_sn2;
|
|
|
|
|
|
xpc_send_msg = xpc_send_msg_sn2;
|
|
|
xpc_received_msg = xpc_received_msg_sn2;
|