|
Thomas Bogendoerfer |
5a0590 |
From: Kamenee Arumugam <kamenee.arumugam@intel.com>
|
|
Thomas Bogendoerfer |
5a0590 |
Date: Thu, 1 Feb 2018 10:52:28 -0800
|
|
Thomas Bogendoerfer |
5a0590 |
Subject: IB/hfi1: Convert PortXmitWait/PortVLXmitWait counters to flit times
|
|
Thomas Bogendoerfer |
5a0590 |
Patch-mainline: v4.16-rc1
|
|
Thomas Bogendoerfer |
5a0590 |
Git-commit: 0719007663ce2d5da653ec1dc3bcfe2ab681b964
|
|
Thomas Bogendoerfer |
5a0590 |
References: bsc#1096793 FATE#325050
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
HFI's counters SendWaitCnt and SendWaitVlCnt are in units
|
|
Thomas Bogendoerfer |
5a0590 |
of TXE cycle time (at 805MHz). OPA counters PortXmitWait and
|
|
Thomas Bogendoerfer |
5a0590 |
PortVLXmtWait are in units of flit times.
|
|
Thomas Bogendoerfer |
5a0590 |
Convert the counter values to flit units using following
|
|
Thomas Bogendoerfer |
5a0590 |
conversion formula:
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
PortXmitWait =
|
|
Thomas Bogendoerfer |
5a0590 |
SendWaitCnt * 2 * (4 /link_width) * (25 Gbps /link_speed)
|
|
Thomas Bogendoerfer |
5a0590 |
PortVLXmitWait =
|
|
Thomas Bogendoerfer |
5a0590 |
SendWaitVLCnt * 2 * (4 /link_width) * (25 Gbps /link_speed)
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
At link up or downgrade events, the link width can change. To ensure
|
|
Thomas Bogendoerfer |
5a0590 |
accurate counter calculations, sample the counters after the events,
|
|
Thomas Bogendoerfer |
5a0590 |
during counter requests, and then aggregate the OPA counters.
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
Reviewed-by: Michael J. Ruhl <michael.j.ruhl@intel.com>
|
|
Thomas Bogendoerfer |
5a0590 |
Signed-off-by: Kamenee Arumugam <kamenee.arumugam@intel.com>
|
|
Thomas Bogendoerfer |
5a0590 |
Signed-off-by: Dennis Dalessandro <dennis.dalessandro@intel.com>
|
|
Thomas Bogendoerfer |
5a0590 |
Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>
|
|
Thomas Bogendoerfer |
5a0590 |
Acked-by: Thomas Bogendoerfer <tbogendoerfer@suse.de>
|
|
Thomas Bogendoerfer |
5a0590 |
---
|
|
Thomas Bogendoerfer |
5a0590 |
drivers/infiniband/hw/hfi1/chip.c | 64 +++++++++++++++++--
|
|
Thomas Bogendoerfer |
5a0590 |
drivers/infiniband/hw/hfi1/chip.h | 4 -
|
|
Thomas Bogendoerfer |
5a0590 |
drivers/infiniband/hw/hfi1/hfi.h | 7 ++
|
|
Thomas Bogendoerfer |
5a0590 |
drivers/infiniband/hw/hfi1/init.c | 9 ++
|
|
Thomas Bogendoerfer |
5a0590 |
drivers/infiniband/hw/hfi1/mad.c | 127 +++++++++++++++++++++++++++++++++++---
|
|
Thomas Bogendoerfer |
5a0590 |
drivers/infiniband/hw/hfi1/mad.h | 47 +++++++++++++-
|
|
Thomas Bogendoerfer |
5a0590 |
6 files changed, 239 insertions(+), 19 deletions(-)
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
--- a/drivers/infiniband/hw/hfi1/chip.c
|
|
Thomas Bogendoerfer |
5a0590 |
+++ b/drivers/infiniband/hw/hfi1/chip.c
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -1083,6 +1083,7 @@ static int qos_rmt_entries(struct hfi1_d
|
|
Thomas Bogendoerfer |
5a0590 |
static void clear_full_mgmt_pkey(struct hfi1_pportdata *ppd);
|
|
Thomas Bogendoerfer |
5a0590 |
static int wait_link_transfer_active(struct hfi1_devdata *dd, int wait_ms);
|
|
Thomas Bogendoerfer |
5a0590 |
static void clear_rsm_rule(struct hfi1_devdata *dd, u8 rule_index);
|
|
Thomas Bogendoerfer |
5a0590 |
+static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width);
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
/*
|
|
Thomas Bogendoerfer |
5a0590 |
* Error interrupt table entry. This is used as input to the interrupt
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -6906,6 +6907,32 @@ void handle_freeze(struct work_struct *w
|
|
Thomas Bogendoerfer |
5a0590 |
/* no longer frozen */
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
+/**
|
|
Thomas Bogendoerfer |
5a0590 |
+ * update_xmit_counters - update PortXmitWait/PortVlXmitWait
|
|
Thomas Bogendoerfer |
5a0590 |
+ * counters.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @ppd: info of physical Hfi port
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @link_width: new link width after link up or downgrade
|
|
Thomas Bogendoerfer |
5a0590 |
+ *
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Update the PortXmitWait and PortVlXmitWait counters after
|
|
Thomas Bogendoerfer |
5a0590 |
+ * a link up or downgrade event to reflect a link width change.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+static void update_xmit_counters(struct hfi1_pportdata *ppd, u16 link_width)
|
|
Thomas Bogendoerfer |
5a0590 |
+{
|
|
Thomas Bogendoerfer |
5a0590 |
+ int i;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 tx_width;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 link_speed;
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ tx_width = tx_link_width(link_width);
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_speed = get_link_speed(ppd->link_speed_active);
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ /*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * There are C_VL_COUNT number of PortVLXmitWait counters.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+ for (i = 0; i < C_VL_COUNT + 1; i++)
|
|
Thomas Bogendoerfer |
5a0590 |
+ get_xmit_wait_counters(ppd, tx_width, link_speed, i);
|
|
Thomas Bogendoerfer |
5a0590 |
+}
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
/*
|
|
Thomas Bogendoerfer |
5a0590 |
* Handle a link up interrupt from the 8051.
|
|
Thomas Bogendoerfer |
5a0590 |
*
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -7527,18 +7554,29 @@ void handle_verify_cap(struct work_struc
|
|
Thomas Bogendoerfer |
5a0590 |
set_link_state(ppd, HLS_GOING_UP);
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
-/*
|
|
Thomas Bogendoerfer |
5a0590 |
- * Apply the link width downgrade enabled policy against the current active
|
|
Thomas Bogendoerfer |
5a0590 |
- * link widths.
|
|
Thomas Bogendoerfer |
5a0590 |
+/**
|
|
Thomas Bogendoerfer |
5a0590 |
+ * apply_link_downgrade_policy - Apply the link width downgrade enabled
|
|
Thomas Bogendoerfer |
5a0590 |
+ * policy against the current active link widths.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @ppd: info of physical Hfi port
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @refresh_widths: True indicates link downgrade event
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @return: True indicates a successful link downgrade. False indicates
|
|
Thomas Bogendoerfer |
5a0590 |
+ * link downgrade event failed and the link will bounce back to
|
|
Thomas Bogendoerfer |
5a0590 |
+ * default link width.
|
|
Thomas Bogendoerfer |
5a0590 |
*
|
|
Thomas Bogendoerfer |
5a0590 |
- * Called when the enabled policy changes or the active link widths change.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Called when the enabled policy changes or the active link widths
|
|
Thomas Bogendoerfer |
5a0590 |
+ * change.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Refresh_widths indicates that a link downgrade occurred. The
|
|
Thomas Bogendoerfer |
5a0590 |
+ * link_downgraded variable is set by refresh_widths and
|
|
Thomas Bogendoerfer |
5a0590 |
+ * determines the success/failure of the policy application.
|
|
Thomas Bogendoerfer |
5a0590 |
*/
|
|
Thomas Bogendoerfer |
5a0590 |
-void apply_link_downgrade_policy(struct hfi1_pportdata *ppd, int refresh_widths)
|
|
Thomas Bogendoerfer |
5a0590 |
+bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
|
|
Thomas Bogendoerfer |
5a0590 |
+ bool refresh_widths)
|
|
Thomas Bogendoerfer |
5a0590 |
{
|
|
Thomas Bogendoerfer |
5a0590 |
int do_bounce = 0;
|
|
Thomas Bogendoerfer |
5a0590 |
int tries;
|
|
Thomas Bogendoerfer |
5a0590 |
u16 lwde;
|
|
Thomas Bogendoerfer |
5a0590 |
u16 tx, rx;
|
|
Thomas Bogendoerfer |
5a0590 |
+ bool link_downgraded = refresh_widths;
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
/* use the hls lock to avoid a race with actual link up */
|
|
Thomas Bogendoerfer |
5a0590 |
tries = 0;
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -7572,6 +7610,7 @@ retry:
|
|
Thomas Bogendoerfer |
5a0590 |
ppd->link_width_downgrade_rx_active == 0) {
|
|
Thomas Bogendoerfer |
5a0590 |
/* the 8051 reported a dead link as a downgrade */
|
|
Thomas Bogendoerfer |
5a0590 |
dd_dev_err(ppd->dd, "Link downgrade is really a link down, ignoring\n");
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_downgraded = false;
|
|
Thomas Bogendoerfer |
5a0590 |
} else if (lwde == 0) {
|
|
Thomas Bogendoerfer |
5a0590 |
/* downgrade is disabled */
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -7588,6 +7627,7 @@ retry:
|
|
Thomas Bogendoerfer |
5a0590 |
ppd->link_width_downgrade_tx_active,
|
|
Thomas Bogendoerfer |
5a0590 |
ppd->link_width_downgrade_rx_active);
|
|
Thomas Bogendoerfer |
5a0590 |
do_bounce = 1;
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_downgraded = false;
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
} else if ((lwde & ppd->link_width_downgrade_tx_active) == 0 ||
|
|
Thomas Bogendoerfer |
5a0590 |
(lwde & ppd->link_width_downgrade_rx_active) == 0) {
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -7599,6 +7639,7 @@ retry:
|
|
Thomas Bogendoerfer |
5a0590 |
lwde, ppd->link_width_downgrade_tx_active,
|
|
Thomas Bogendoerfer |
5a0590 |
ppd->link_width_downgrade_rx_active);
|
|
Thomas Bogendoerfer |
5a0590 |
do_bounce = 1;
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_downgraded = false;
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
done:
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -7610,6 +7651,8 @@ done:
|
|
Thomas Bogendoerfer |
5a0590 |
set_link_state(ppd, HLS_DN_OFFLINE);
|
|
Thomas Bogendoerfer |
5a0590 |
start_link(ppd);
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ return link_downgraded;
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
/*
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -7623,7 +7666,8 @@ void handle_link_downgrade(struct work_s
|
|
Thomas Bogendoerfer |
5a0590 |
link_downgrade_work);
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
dd_dev_info(ppd->dd, "8051: Link width downgrade\n");
|
|
Thomas Bogendoerfer |
5a0590 |
- apply_link_downgrade_policy(ppd, 1);
|
|
Thomas Bogendoerfer |
5a0590 |
+ if (apply_link_downgrade_policy(ppd, true))
|
|
Thomas Bogendoerfer |
5a0590 |
+ update_xmit_counters(ppd, ppd->link_width_downgrade_tx_active);
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
static char *dcc_err_string(char *buf, int buf_len, u64 flags)
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -10594,6 +10638,14 @@ int set_link_state(struct hfi1_pportdata
|
|
Thomas Bogendoerfer |
5a0590 |
add_rcvctrl(dd, RCV_CTRL_RCV_PORT_ENABLE_SMASK);
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
handle_linkup_change(dd, 1);
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ /*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * After link up, a new link width will have been set.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Update the xmit counters with regards to the new
|
|
Thomas Bogendoerfer |
5a0590 |
+ * link width.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+ update_xmit_counters(ppd, ppd->link_width_active);
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
ppd->host_link_state = HLS_UP_INIT;
|
|
Thomas Bogendoerfer |
5a0590 |
update_statusp(ppd, IB_PORT_INIT);
|
|
Thomas Bogendoerfer |
5a0590 |
break;
|
|
Thomas Bogendoerfer |
5a0590 |
--- a/drivers/infiniband/hw/hfi1/chip.h
|
|
Thomas Bogendoerfer |
5a0590 |
+++ b/drivers/infiniband/hw/hfi1/chip.h
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -736,8 +736,8 @@ int read_8051_config(struct hfi1_devdata
|
|
Thomas Bogendoerfer |
5a0590 |
int start_link(struct hfi1_pportdata *ppd);
|
|
Thomas Bogendoerfer |
5a0590 |
int bringup_serdes(struct hfi1_pportdata *ppd);
|
|
Thomas Bogendoerfer |
5a0590 |
void set_intr_state(struct hfi1_devdata *dd, u32 enable);
|
|
Thomas Bogendoerfer |
5a0590 |
-void apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
|
|
Thomas Bogendoerfer |
5a0590 |
- int refresh_widths);
|
|
Thomas Bogendoerfer |
5a0590 |
+bool apply_link_downgrade_policy(struct hfi1_pportdata *ppd,
|
|
Thomas Bogendoerfer |
5a0590 |
+ bool refresh_widths);
|
|
Thomas Bogendoerfer |
5a0590 |
void update_usrhead(struct hfi1_ctxtdata *rcd, u32 hd, u32 updegr, u32 egrhd,
|
|
Thomas Bogendoerfer |
5a0590 |
u32 intr_adjust, u32 npkts);
|
|
Thomas Bogendoerfer |
5a0590 |
int stop_drain_data_vls(struct hfi1_devdata *dd);
|
|
Thomas Bogendoerfer |
5a0590 |
--- a/drivers/infiniband/hw/hfi1/hfi.h
|
|
Thomas Bogendoerfer |
5a0590 |
+++ b/drivers/infiniband/hw/hfi1/hfi.h
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -858,6 +858,13 @@ struct hfi1_pportdata {
|
|
Thomas Bogendoerfer |
5a0590 |
struct work_struct linkstate_active_work;
|
|
Thomas Bogendoerfer |
5a0590 |
/* Does this port need to prescan for FECNs */
|
|
Thomas Bogendoerfer |
5a0590 |
bool cc_prescan;
|
|
Thomas Bogendoerfer |
5a0590 |
+ /*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Sample sendWaitCnt & sendWaitVlCnt during link transition
|
|
Thomas Bogendoerfer |
5a0590 |
+ * and counter request.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+ u64 port_vl_xmit_wait_last[C_VL_COUNT + 1];
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 prev_link_width;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u64 vl_xmit_flit_cnt[C_VL_COUNT + 1];
|
|
Thomas Bogendoerfer |
5a0590 |
};
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
typedef int (*rhf_rcv_function_ptr)(struct hfi1_packet *packet);
|
|
Thomas Bogendoerfer |
5a0590 |
--- a/drivers/infiniband/hw/hfi1/init.c
|
|
Thomas Bogendoerfer |
5a0590 |
+++ b/drivers/infiniband/hw/hfi1/init.c
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -637,6 +637,15 @@ void hfi1_init_pportdata(struct pci_dev
|
|
Thomas Bogendoerfer |
5a0590 |
ppd->dd = dd;
|
|
Thomas Bogendoerfer |
5a0590 |
ppd->hw_pidx = hw_pidx;
|
|
Thomas Bogendoerfer |
5a0590 |
ppd->port = port; /* IB port number, not index */
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->prev_link_width = LINK_WIDTH_DEFAULT;
|
|
Thomas Bogendoerfer |
5a0590 |
+ /*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * There are C_VL_COUNT number of PortVLXmitWait counters.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Adding 1 to C_VL_COUNT to include the PortXmitWait counter.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+ for (i = 0; i < C_VL_COUNT + 1; i++) {
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->port_vl_xmit_wait_last[i] = 0;
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->vl_xmit_flit_cnt[i] = 0;
|
|
Thomas Bogendoerfer |
5a0590 |
+ }
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
default_pkey_idx = 1;
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
--- a/drivers/infiniband/hw/hfi1/mad.c
|
|
Thomas Bogendoerfer |
5a0590 |
+++ b/drivers/infiniband/hw/hfi1/mad.c
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -2648,6 +2648,79 @@ static void a0_portstatus(struct hfi1_pp
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
}
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
+/**
|
|
Thomas Bogendoerfer |
5a0590 |
+ * tx_link_width - convert link width bitmask to integer
|
|
Thomas Bogendoerfer |
5a0590 |
+ * value representing actual link width.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @link_width: width of active link
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @return: return index of the bit set in link_width var
|
|
Thomas Bogendoerfer |
5a0590 |
+ *
|
|
Thomas Bogendoerfer |
5a0590 |
+ * The function convert and return the index of bit set
|
|
Thomas Bogendoerfer |
5a0590 |
+ * that indicate the current link width.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+u16 tx_link_width(u16 link_width)
|
|
Thomas Bogendoerfer |
5a0590 |
+{
|
|
Thomas Bogendoerfer |
5a0590 |
+ int n = LINK_WIDTH_DEFAULT;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 tx_width = n;
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ while (link_width && n) {
|
|
Thomas Bogendoerfer |
5a0590 |
+ if (link_width & (1 << (n - 1))) {
|
|
Thomas Bogendoerfer |
5a0590 |
+ tx_width = n;
|
|
Thomas Bogendoerfer |
5a0590 |
+ break;
|
|
Thomas Bogendoerfer |
5a0590 |
+ }
|
|
Thomas Bogendoerfer |
5a0590 |
+ n--;
|
|
Thomas Bogendoerfer |
5a0590 |
+ }
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ return tx_width;
|
|
Thomas Bogendoerfer |
5a0590 |
+}
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+/**
|
|
Thomas Bogendoerfer |
5a0590 |
+ * get_xmit_wait_counters - Convert HFI 's SendWaitCnt/SendWaitVlCnt
|
|
Thomas Bogendoerfer |
5a0590 |
+ * counter in unit of TXE cycle times to flit times.
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @ppd: info of physical Hfi port
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @link_width: width of active link
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @link_speed: speed of active link
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @vl: represent VL0-VL7, VL15 for PortVLXmitWait counters request
|
|
Thomas Bogendoerfer |
5a0590 |
+ * and if vl value is C_VL_COUNT, it represent SendWaitCnt
|
|
Thomas Bogendoerfer |
5a0590 |
+ * counter request
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @return: return SendWaitCnt/SendWaitVlCnt counter value per vl.
|
|
Thomas Bogendoerfer |
5a0590 |
+ *
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Convert SendWaitCnt/SendWaitVlCnt counter from TXE cycle times to
|
|
Thomas Bogendoerfer |
5a0590 |
+ * flit times. Call this function to samples these counters. This
|
|
Thomas Bogendoerfer |
5a0590 |
+ * function will calculate for previous state transition and update
|
|
Thomas Bogendoerfer |
5a0590 |
+ * current state at end of function using ppd->prev_link_width and
|
|
Thomas Bogendoerfer |
5a0590 |
+ * ppd->port_vl_xmit_wait_last to port_vl_xmit_wait_curr and link_width.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd,
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 link_width, u16 link_speed, int vl)
|
|
Thomas Bogendoerfer |
5a0590 |
+{
|
|
Thomas Bogendoerfer |
5a0590 |
+ u64 port_vl_xmit_wait_curr;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u64 delta_vl_xmit_wait;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u64 xmit_wait_val;
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ if (vl > C_VL_COUNT)
|
|
Thomas Bogendoerfer |
5a0590 |
+ return 0;
|
|
Thomas Bogendoerfer |
5a0590 |
+ if (vl < C_VL_COUNT)
|
|
Thomas Bogendoerfer |
5a0590 |
+ port_vl_xmit_wait_curr =
|
|
Thomas Bogendoerfer |
5a0590 |
+ read_port_cntr(ppd, C_TX_WAIT_VL, vl);
|
|
Thomas Bogendoerfer |
5a0590 |
+ else
|
|
Thomas Bogendoerfer |
5a0590 |
+ port_vl_xmit_wait_curr =
|
|
Thomas Bogendoerfer |
5a0590 |
+ read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL);
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ xmit_wait_val =
|
|
Thomas Bogendoerfer |
5a0590 |
+ port_vl_xmit_wait_curr -
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->port_vl_xmit_wait_last[vl];
|
|
Thomas Bogendoerfer |
5a0590 |
+ delta_vl_xmit_wait =
|
|
Thomas Bogendoerfer |
5a0590 |
+ convert_xmit_counter(xmit_wait_val,
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->prev_link_width,
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_speed);
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->vl_xmit_flit_cnt[vl] += delta_vl_xmit_wait;
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->port_vl_xmit_wait_last[vl] = port_vl_xmit_wait_curr;
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->prev_link_width = link_width;
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+ return ppd->vl_xmit_flit_cnt[vl];
|
|
Thomas Bogendoerfer |
5a0590 |
+}
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
static int pma_get_opa_portstatus(struct opa_pma_mad *pmp,
|
|
Thomas Bogendoerfer |
5a0590 |
struct ib_device *ibdev,
|
|
Thomas Bogendoerfer |
5a0590 |
u8 port, u32 *resp_len)
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -2667,6 +2740,8 @@ static int pma_get_opa_portstatus(struct
|
|
Thomas Bogendoerfer |
5a0590 |
struct hfi1_pportdata *ppd = ppd_from_ibp(ibp);
|
|
Thomas Bogendoerfer |
5a0590 |
int vfi;
|
|
Thomas Bogendoerfer |
5a0590 |
u64 tmp, tmp2;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 link_width;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 link_speed;
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
response_data_size = sizeof(struct opa_port_status_rsp) +
|
|
Thomas Bogendoerfer |
5a0590 |
num_vls * sizeof(struct _vls_pctrs);
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -2710,8 +2785,16 @@ static int pma_get_opa_portstatus(struct
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->port_multicast_rcv_pkts =
|
|
Thomas Bogendoerfer |
5a0590 |
cpu_to_be64(read_dev_cntr(dd, C_DC_MC_RCV_PKTS,
|
|
Thomas Bogendoerfer |
5a0590 |
CNTR_INVALID_VL));
|
|
Thomas Bogendoerfer |
5a0590 |
+ /*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Convert PortXmitWait counter from TXE cycle times
|
|
Thomas Bogendoerfer |
5a0590 |
+ * to flit times.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_width =
|
|
Thomas Bogendoerfer |
5a0590 |
+ tx_link_width(ppd->link_width_downgrade_tx_active);
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_speed = get_link_speed(ppd->link_speed_active);
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->port_xmit_wait =
|
|
Thomas Bogendoerfer |
5a0590 |
- cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL));
|
|
Thomas Bogendoerfer |
5a0590 |
+ cpu_to_be64(get_xmit_wait_counters(ppd, link_width,
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_speed, C_VL_COUNT));
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->port_rcv_fecn =
|
|
Thomas Bogendoerfer |
5a0590 |
cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN, CNTR_INVALID_VL));
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->port_rcv_becn =
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -2776,10 +2859,14 @@ static int pma_get_opa_portstatus(struct
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->vls[vfi].port_vl_xmit_pkts =
|
|
Thomas Bogendoerfer |
5a0590 |
cpu_to_be64(read_port_cntr(ppd, C_TX_PKT_VL,
|
|
Thomas Bogendoerfer |
5a0590 |
idx_from_vl(vl)));
|
|
Thomas Bogendoerfer |
5a0590 |
-
|
|
Thomas Bogendoerfer |
5a0590 |
+ /*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Convert PortVlXmitWait counter from TXE cycle
|
|
Thomas Bogendoerfer |
5a0590 |
+ * times to flit times.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->vls[vfi].port_vl_xmit_wait =
|
|
Thomas Bogendoerfer |
5a0590 |
- cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT_VL,
|
|
Thomas Bogendoerfer |
5a0590 |
- idx_from_vl(vl)));
|
|
Thomas Bogendoerfer |
5a0590 |
+ cpu_to_be64(get_xmit_wait_counters(ppd, link_width,
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_speed,
|
|
Thomas Bogendoerfer |
5a0590 |
+ idx_from_vl(vl)));
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->vls[vfi].port_vl_rcv_fecn =
|
|
Thomas Bogendoerfer |
5a0590 |
cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN_VL,
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -2910,6 +2997,8 @@ static int pma_get_opa_datacounters(stru
|
|
Thomas Bogendoerfer |
5a0590 |
unsigned long vl;
|
|
Thomas Bogendoerfer |
5a0590 |
u32 vl_select_mask;
|
|
Thomas Bogendoerfer |
5a0590 |
int vfi;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 link_width;
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 link_speed;
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
num_ports = be32_to_cpu(pmp->mad_hdr.attr_mod) >> 24;
|
|
Thomas Bogendoerfer |
5a0590 |
num_pslm = hweight64(be64_to_cpu(req->port_select_mask[3]));
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -2960,8 +3049,16 @@ static int pma_get_opa_datacounters(stru
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->link_quality_indicator = cpu_to_be32((u32)lq);
|
|
Thomas Bogendoerfer |
5a0590 |
pma_get_opa_port_dctrs(ibdev, rsp);
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
+ /*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Convert PortXmitWait counter from TXE
|
|
Thomas Bogendoerfer |
5a0590 |
+ * cycle times to flit times.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_width =
|
|
Thomas Bogendoerfer |
5a0590 |
+ tx_link_width(ppd->link_width_downgrade_tx_active);
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_speed = get_link_speed(ppd->link_speed_active);
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->port_xmit_wait =
|
|
Thomas Bogendoerfer |
5a0590 |
- cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL));
|
|
Thomas Bogendoerfer |
5a0590 |
+ cpu_to_be64(get_xmit_wait_counters(ppd, link_width,
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_speed, C_VL_COUNT));
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->port_rcv_fecn =
|
|
Thomas Bogendoerfer |
5a0590 |
cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN, CNTR_INVALID_VL));
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->port_rcv_becn =
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -2997,9 +3094,14 @@ static int pma_get_opa_datacounters(stru
|
|
Thomas Bogendoerfer |
5a0590 |
cpu_to_be64(read_dev_cntr(dd, C_DC_RX_PKT_VL,
|
|
Thomas Bogendoerfer |
5a0590 |
idx_from_vl(vl)));
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
+ /*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * Convert PortVlXmitWait counter from TXE
|
|
Thomas Bogendoerfer |
5a0590 |
+ * cycle times to flit times.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->vls[vfi].port_vl_xmit_wait =
|
|
Thomas Bogendoerfer |
5a0590 |
- cpu_to_be64(read_port_cntr(ppd, C_TX_WAIT_VL,
|
|
Thomas Bogendoerfer |
5a0590 |
- idx_from_vl(vl)));
|
|
Thomas Bogendoerfer |
5a0590 |
+ cpu_to_be64(get_xmit_wait_counters(ppd, link_width,
|
|
Thomas Bogendoerfer |
5a0590 |
+ link_speed,
|
|
Thomas Bogendoerfer |
5a0590 |
+ idx_from_vl(vl)));
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
rsp->vls[vfi].port_vl_rcv_fecn =
|
|
Thomas Bogendoerfer |
5a0590 |
cpu_to_be64(read_dev_cntr(dd, C_DC_RCV_FCN_VL,
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -3417,9 +3519,11 @@ static int pma_set_opa_portstatus(struct
|
|
Thomas Bogendoerfer |
5a0590 |
if (counter_select & CS_PORT_MCAST_RCV_PKTS)
|
|
Thomas Bogendoerfer |
5a0590 |
write_dev_cntr(dd, C_DC_MC_RCV_PKTS, CNTR_INVALID_VL, 0);
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
- if (counter_select & CS_PORT_XMIT_WAIT)
|
|
Thomas Bogendoerfer |
5a0590 |
+ if (counter_select & CS_PORT_XMIT_WAIT) {
|
|
Thomas Bogendoerfer |
5a0590 |
write_port_cntr(ppd, C_TX_WAIT, CNTR_INVALID_VL, 0);
|
|
Thomas Bogendoerfer |
5a0590 |
-
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->port_vl_xmit_wait_last[C_VL_COUNT] = 0;
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->vl_xmit_flit_cnt[C_VL_COUNT] = 0;
|
|
Thomas Bogendoerfer |
5a0590 |
+ }
|
|
Thomas Bogendoerfer |
5a0590 |
/* ignore cs_sw_portCongestion for HFIs */
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
if (counter_select & CS_PORT_RCV_FECN)
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -3492,8 +3596,11 @@ static int pma_set_opa_portstatus(struct
|
|
Thomas Bogendoerfer |
5a0590 |
if (counter_select & CS_PORT_RCV_PKTS)
|
|
Thomas Bogendoerfer |
5a0590 |
write_dev_cntr(dd, C_DC_RX_PKT_VL, idx_from_vl(vl), 0);
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
- if (counter_select & CS_PORT_XMIT_WAIT)
|
|
Thomas Bogendoerfer |
5a0590 |
+ if (counter_select & CS_PORT_XMIT_WAIT) {
|
|
Thomas Bogendoerfer |
5a0590 |
write_port_cntr(ppd, C_TX_WAIT_VL, idx_from_vl(vl), 0);
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->port_vl_xmit_wait_last[idx_from_vl(vl)] = 0;
|
|
Thomas Bogendoerfer |
5a0590 |
+ ppd->vl_xmit_flit_cnt[idx_from_vl(vl)] = 0;
|
|
Thomas Bogendoerfer |
5a0590 |
+ }
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
/* sw_port_vl_congestion is 0 for HFIs */
|
|
Thomas Bogendoerfer |
5a0590 |
if (counter_select & CS_PORT_RCV_FECN)
|
|
Thomas Bogendoerfer |
5a0590 |
--- a/drivers/infiniband/hw/hfi1/mad.h
|
|
Thomas Bogendoerfer |
5a0590 |
+++ b/drivers/infiniband/hw/hfi1/mad.h
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -180,6 +180,15 @@ struct opa_mad_notice_attr {
|
|
Thomas Bogendoerfer |
5a0590 |
#define OPA_VLARB_PREEMPT_MATRIX 3
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
#define IB_PMA_PORT_COUNTERS_CONG cpu_to_be16(0xFF00)
|
|
Thomas Bogendoerfer |
5a0590 |
+#define LINK_SPEED_25G 1
|
|
Thomas Bogendoerfer |
5a0590 |
+#define LINK_SPEED_12_5G 2
|
|
Thomas Bogendoerfer |
5a0590 |
+#define LINK_WIDTH_DEFAULT 4
|
|
Thomas Bogendoerfer |
5a0590 |
+#define DECIMAL_FACTORING 1000
|
|
Thomas Bogendoerfer |
5a0590 |
+/*
|
|
Thomas Bogendoerfer |
5a0590 |
+ * The default link width is multiplied by 1000
|
|
Thomas Bogendoerfer |
5a0590 |
+ * to get accurate value after division.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+#define FACTOR_LINK_WIDTH (LINK_WIDTH_DEFAULT * DECIMAL_FACTORING)
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
struct ib_pma_portcounters_cong {
|
|
Thomas Bogendoerfer |
5a0590 |
u8 reserved;
|
|
Thomas Bogendoerfer |
5a0590 |
@@ -429,5 +438,41 @@ struct sc2vlnt {
|
|
Thomas Bogendoerfer |
5a0590 |
|
|
Thomas Bogendoerfer |
5a0590 |
void hfi1_event_pkey_change(struct hfi1_devdata *dd, u8 port);
|
|
Thomas Bogendoerfer |
5a0590 |
void hfi1_handle_trap_timer(unsigned long data);
|
|
Thomas Bogendoerfer |
5a0590 |
-
|
|
Thomas Bogendoerfer |
5a0590 |
+u16 tx_link_width(u16 link_width);
|
|
Thomas Bogendoerfer |
5a0590 |
+u64 get_xmit_wait_counters(struct hfi1_pportdata *ppd, u16 link_width,
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 link_speed, int vl);
|
|
Thomas Bogendoerfer |
5a0590 |
+/**
|
|
Thomas Bogendoerfer |
5a0590 |
+ * get_link_speed - determine whether 12.5G or 25G speed
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @link_speed: the speed of active link
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @return: Return 2 if link speed identified as 12.5G
|
|
Thomas Bogendoerfer |
5a0590 |
+ * or return 1 if link speed is 25G.
|
|
Thomas Bogendoerfer |
5a0590 |
+ *
|
|
Thomas Bogendoerfer |
5a0590 |
+ * The function indirectly calculate required link speed
|
|
Thomas Bogendoerfer |
5a0590 |
+ * value for convert_xmit_counter function. If the link
|
|
Thomas Bogendoerfer |
5a0590 |
+ * speed is 25G, the function return as 1 as it is required
|
|
Thomas Bogendoerfer |
5a0590 |
+ * by xmit counter conversion formula :-( 25G / link_speed).
|
|
Thomas Bogendoerfer |
5a0590 |
+ * This conversion will provide value 1 if current
|
|
Thomas Bogendoerfer |
5a0590 |
+ * link speed is 25G or 2 if 12.5G.This is done to avoid
|
|
Thomas Bogendoerfer |
5a0590 |
+ * 12.5 float number conversion.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+static inline u16 get_link_speed(u16 link_speed)
|
|
Thomas Bogendoerfer |
5a0590 |
+{
|
|
Thomas Bogendoerfer |
5a0590 |
+ return (link_speed == 1) ?
|
|
Thomas Bogendoerfer |
5a0590 |
+ LINK_SPEED_12_5G : LINK_SPEED_25G;
|
|
Thomas Bogendoerfer |
5a0590 |
+}
|
|
Thomas Bogendoerfer |
5a0590 |
+
|
|
Thomas Bogendoerfer |
5a0590 |
+/**
|
|
Thomas Bogendoerfer |
5a0590 |
+ * convert_xmit_counter - calculate flit times for given xmit counter
|
|
Thomas Bogendoerfer |
5a0590 |
+ * value
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @xmit_wait_val: current xmit counter value
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @link_width: width of active link
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @link_speed: speed of active link
|
|
Thomas Bogendoerfer |
5a0590 |
+ * @return: return xmit counter value in flit times.
|
|
Thomas Bogendoerfer |
5a0590 |
+ */
|
|
Thomas Bogendoerfer |
5a0590 |
+static inline u64 convert_xmit_counter(u64 xmit_wait_val, u16 link_width,
|
|
Thomas Bogendoerfer |
5a0590 |
+ u16 link_speed)
|
|
Thomas Bogendoerfer |
5a0590 |
+{
|
|
Thomas Bogendoerfer |
5a0590 |
+ return (xmit_wait_val * 2 * (FACTOR_LINK_WIDTH / link_width)
|
|
Thomas Bogendoerfer |
5a0590 |
+ * link_speed) / DECIMAL_FACTORING;
|
|
Thomas Bogendoerfer |
5a0590 |
+}
|
|
Thomas Bogendoerfer |
5a0590 |
#endif /* _HFI1_MAD_H */
|