From ef10c0dad9bbb2a75d025f29209b4952fedcabc2 Mon Sep 17 00:00:00 2001
From: Ben Skeggs <bskeggs@redhat.com>
Date: Tue, 21 Jul 2020 11:34:07 +1000
Subject: drm/nouveau/kms/nv50-: implement proper push buffer control logic
Git-commit: 0a96099691c8cd1ac0744ef30b6846869dc2b566
Patch-mainline: v5.9-rc1
References: jsc#SLE-12680, jsc#SLE-12880, jsc#SLE-12882, jsc#SLE-12883, jsc#SLE-13496, jsc#SLE-15322
We had a, what was supposed to be temporary, hack in the KMS code where we'd
completely drain an EVO/NVD channel's push buffer when wrapping to the start
again, instead of treating it as a ring buffer.
Let's fix that, finally.
Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Signed-off-by: Patrik Jakobsson <pjakobsson@suse.de>
---
drivers/gpu/drm/nouveau/dispnv50/disp.c | 157 ++++++++++--------
drivers/gpu/drm/nouveau/dispnv50/disp.h | 4 +
drivers/gpu/drm/nouveau/include/nvif/object.h | 13 ++
.../drm/nouveau/nvkm/engine/disp/channv50.h | 2 +
.../drm/nouveau/nvkm/engine/disp/coregf119.c | 4 +-
.../drm/nouveau/nvkm/engine/disp/coregp102.c | 2 +-
.../drm/nouveau/nvkm/engine/disp/coregv100.c | 3 +-
.../drm/nouveau/nvkm/engine/disp/corenv50.c | 4 +-
.../drm/nouveau/nvkm/engine/disp/dmacgf119.c | 4 +-
.../drm/nouveau/nvkm/engine/disp/dmacgp102.c | 2 +-
.../drm/nouveau/nvkm/engine/disp/dmacgv100.c | 4 +-
.../drm/nouveau/nvkm/engine/disp/dmacnv50.c | 4 +-
12 files changed, 125 insertions(+), 78 deletions(-)
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c
index b993083e4c86..b34cad030e99 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c
@@ -41,6 +41,8 @@
#include <drm/drm_scdc_helper.h>
#include <drm/drm_vblank.h>
+#include <nvif/push507c.h>
+
#include <nvif/class.h>
#include <nvif/cl0002.h>
#include <nvif/cl5070.h>
@@ -48,6 +50,8 @@
#include <nvif/event.h>
#include <nvif/timer.h>
+#include <nvhw/class/cl507c.h>
+
#include "nouveau_drv.h"
#include "nouveau_dma.h"
#include "nouveau_gem.h"
@@ -120,21 +124,93 @@ static void
nv50_dmac_kick(struct nvif_push *push)
{
struct nv50_dmac *dmac = container_of(push, typeof(*dmac), _push);
- evo_kick(push->cur, dmac);
- push->bgn = push->cur = push->end;
+
+ dmac->cur = push->cur - (u32 *)dmac->_push.mem.object.map.ptr;
+ if (dmac->put != dmac->cur) {
+ /* Push buffer fetches are not coherent with BAR1, we need to ensure
+ * writes have been flushed right through to VRAM before writing PUT.
+ */
+ if (dmac->push->mem.type & NVIF_MEM_VRAM) {
+ struct nvif_device *device = dmac->base.device;
+ nvif_wr32(&device->object, 0x070000, 0x00000001);
+ nvif_msec(device, 2000,
+ if (!(nvif_rd32(&device->object, 0x070000) & 0x00000002))
+ break;
+ );
+ }
+
+ NVIF_WV32(&dmac->base.user, NV507C, PUT, PTR, dmac->cur);
+ dmac->put = dmac->cur;
+ }
+
+ push->bgn = push->cur;
+}
+
+static int
+nv50_dmac_free(struct nv50_dmac *dmac)
+{
+ u32 get = NVIF_RV32(&dmac->base.user, NV507C, GET, PTR);
+ if (get > dmac->cur) /* NVIDIA stay 5 away from GET, do the same. */
+ return get - dmac->cur - 5;
+ return dmac->max - dmac->cur;
+}
+
+static int
+nv50_dmac_wind(struct nv50_dmac *dmac)
+{
+ /* Wait for GET to depart from the beginning of the push buffer to
+ * prevent writing PUT == GET, which would be ignored by HW.
+ */
+ u32 get = NVIF_RV32(&dmac->base.user, NV507C, GET, PTR);
+ if (get == 0) {
+ /* Corner-case, HW idle, but non-committed work pending. */
+ if (dmac->put == 0)
+ nv50_dmac_kick(dmac->push);
+
+ if (nvif_msec(dmac->base.device, 2000,
+ if (NVIF_TV32(&dmac->base.user, NV507C, GET, PTR, >, 0))
+ break;
+ ) < 0)
+ return -ETIMEDOUT;
+ }
+
+ PUSH_RSVD(dmac->push, PUSH_JUMP(dmac->push, 0));
+ dmac->cur = 0;
+ return 0;
}
static int
nv50_dmac_wait(struct nvif_push *push, u32 size)
{
struct nv50_dmac *dmac = container_of(push, typeof(*dmac), _push);
- u32 *ptr = evo_wait(dmac, size);
- if (!ptr)
+ int free;
+
+ if (WARN_ON(size > dmac->max))
+ return -EINVAL;
+
+ dmac->cur = push->cur - (u32 *)dmac->_push.mem.object.map.ptr;
+ if (dmac->cur + size >= dmac->max) {
+ int ret = nv50_dmac_wind(dmac);
+ if (ret)
+ return ret;
+
+ push->cur = dmac->_push.mem.object.map.ptr;
+ push->cur = push->cur + dmac->cur;
+ nv50_dmac_kick(push);
+ }
+
+ if (nvif_msec(dmac->base.device, 2000,
+ if ((free = nv50_dmac_free(dmac)) >= size)
+ break;
+ ) < 0) {
+ WARN_ON(1);
return -ETIMEDOUT;
+ }
- push->bgn = ptr;
- push->cur = ptr;
- push->end = ptr + size;
+ push->bgn = dmac->_push.mem.object.map.ptr;
+ push->bgn = push->bgn + dmac->cur;
+ push->cur = push->bgn;
+ push->end = push->cur + free;
return 0;
}
@@ -171,6 +247,10 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
dmac->_push.wait = nv50_dmac_wait;
dmac->_push.kick = nv50_dmac_kick;
dmac->push = &dmac->_push;
+ dmac->push->bgn = dmac->_push.mem.object.map.ptr;
+ dmac->push->cur = dmac->push->bgn;
+ dmac->push->end = dmac->push->bgn;
+ dmac->max = 0x1000/4 - 1;
args->pushbuf = nvif_handle(&dmac->_push.mem.object);
@@ -209,69 +289,6 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
return ret;
}
-/******************************************************************************
- * EVO channel helpers
- *****************************************************************************/
-static void
-evo_flush(struct nv50_dmac *dmac)
-{
- /* Push buffer fetches are not coherent with BAR1, we need to ensure
- * writes have been flushed right through to VRAM before writing PUT.
- */
- if (dmac->push->mem.type & NVIF_MEM_VRAM) {
- struct nvif_device *device = dmac->base.device;
- nvif_wr32(&device->object, 0x070000, 0x00000001);
- nvif_msec(device, 2000,
- if (!(nvif_rd32(&device->object, 0x070000) & 0x00000002))
- break;
- );
- }
-}
-
-u32 *
-evo_wait(struct nv50_dmac *evoc, int nr)
-{
- struct nv50_dmac *dmac = evoc;
- struct nvif_device *device = dmac->base.device;
- u32 put;
-
- if (dmac->push->cur != dmac->push->bgn)
- PUSH_KICK(dmac->push);
-
- put = nvif_rd32(&dmac->base.user, 0x0000) / 4;
-
- mutex_lock(&dmac->lock);
- if (put + nr >= (PAGE_SIZE / 4) - 8) {
- dmac->ptr[put] = 0x20000000;
- evo_flush(dmac);
-
- nvif_wr32(&dmac->base.user, 0x0000, 0x00000000);
- if (nvif_msec(device, 2000,
- if (!nvif_rd32(&dmac->base.user, 0x0004))
- break;
- ) < 0) {
- mutex_unlock(&dmac->lock);
- pr_err("nouveau: evo channel stalled\n");
- return NULL;
- }
-
- put = 0;
- }
-
- return dmac->ptr + put;
-}
-
-void
-evo_kick(u32 *push, struct nv50_dmac *evoc)
-{
- struct nv50_dmac *dmac = evoc;
-
- evo_flush(dmac);
-
- nvif_wr32(&dmac->base.user, 0x0000, (push - dmac->ptr) << 2);
- mutex_unlock(&dmac->lock);
-}
-
/******************************************************************************
* Output path helpers
*****************************************************************************/
diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.h b/drivers/gpu/drm/nouveau/dispnv50/disp.h
index fbf573a01f58..92bddc083617 100644
--- a/drivers/gpu/drm/nouveau/dispnv50/disp.h
+++ b/drivers/gpu/drm/nouveau/dispnv50/disp.h
@@ -73,6 +73,10 @@ struct nv50_dmac {
* grabbed by evo_wait (if the pushbuf reservation is successful) and
* dropped again by evo_kick. */
struct mutex lock;
+
+ u32 cur;
+ u32 put;
+ u32 max;
};
struct nv50_outp_atom {
diff --git a/drivers/gpu/drm/nouveau/include/nvif/object.h b/drivers/gpu/drm/nouveau/include/nvif/object.h
index a39d2bf7064b..1e4c158d20fa 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/object.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/object.h
@@ -116,6 +116,19 @@ struct nvif_mclass {
_cid; \
})
+#define NVIF_RD32_(p,o,dr) nvif_rd32((p), (o) + (dr))
+#define NVIF_WR32_(p,o,dr,f) nvif_wr32((p), (o) + (dr), (f))
+#define NVIF_RD32(p,A...) DRF_RD(NVIF_RD32_, (p), 0, ##A)
+#define NVIF_RV32(p,A...) DRF_RV(NVIF_RD32_, (p), 0, ##A)
+#define NVIF_TV32(p,A...) DRF_TV(NVIF_RD32_, (p), 0, ##A)
+#define NVIF_TD32(p,A...) DRF_TD(NVIF_RD32_, (p), 0, ##A)
+#define NVIF_WR32(p,A...) DRF_WR( NVIF_WR32_, (p), 0, ##A)
+#define NVIF_WV32(p,A...) DRF_WV( NVIF_WR32_, (p), 0, ##A)
+#define NVIF_WD32(p,A...) DRF_WD( NVIF_WR32_, (p), 0, ##A)
+#define NVIF_MR32(p,A...) DRF_MR(NVIF_RD32_, NVIF_WR32_, u32, (p), 0, ##A)
+#define NVIF_MV32(p,A...) DRF_MV(NVIF_RD32_, NVIF_WR32_, u32, (p), 0, ##A)
+#define NVIF_MD32(p,A...) DRF_MD(NVIF_RD32_, NVIF_WR32_, u32, (p), 0, ##A)
+
/*XXX*/
#include <core/object.h>
#define nvxx_object(a) ({ \
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
index e55054b7329f..9cf2cfe2010c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
@@ -21,6 +21,8 @@ struct nv50_disp_chan {
struct nvkm_memory *memory;
u64 push;
+
+ u32 suspend_put;
};
struct nv50_disp_chan_func {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
index d162b9cf4eac..689e3cdd959a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
@@ -182,6 +182,8 @@ gf119_disp_core_fini(struct nv50_disp_chan *chan)
nvkm_error(subdev, "core fini: %08x\n",
nvkm_rd32(device, 0x610490));
}
+
+ chan->suspend_put = nvkm_rd32(device, 0x640000);
}
static int
@@ -195,7 +197,7 @@ gf119_disp_core_init(struct nv50_disp_chan *chan)
nvkm_wr32(device, 0x610498, 0x00010000);
nvkm_wr32(device, 0x61049c, 0x00000001);
nvkm_mask(device, 0x610490, 0x00000010, 0x00000010);
- nvkm_wr32(device, 0x640000, 0x00000000);
+ nvkm_wr32(device, 0x640000, chan->suspend_put);
nvkm_wr32(device, 0x610490, 0x01000013);
/* wait for it to go inactive */
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c
index 5b7f993c73c7..1b435beef3bf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c
@@ -36,7 +36,7 @@ gp102_disp_core_init(struct nv50_disp_chan *chan)
nvkm_wr32(device, 0x611498, 0x00010000);
nvkm_wr32(device, 0x61149c, 0x00000001);
nvkm_mask(device, 0x610490, 0x00000010, 0x00000010);
- nvkm_wr32(device, 0x640000, 0x00000000);
+ nvkm_wr32(device, 0x640000, chan->suspend_put);
nvkm_wr32(device, 0x610490, 0x01000013);
/* wait for it to go inactive */
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c
index 4592d0e69fec..e20a48f201f6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c
@@ -167,6 +167,7 @@ gv100_disp_core_fini(struct nv50_disp_chan *chan)
nvkm_mask(device, 0x6104e0, 0x00000010, 0x00000000);
gv100_disp_core_idle(chan);
nvkm_mask(device, 0x6104e0, 0x00000002, 0x00000000);
+ chan->suspend_put = nvkm_rd32(device, 0x680000);
}
static int
@@ -181,7 +182,7 @@ gv100_disp_core_init(struct nv50_disp_chan *chan)
nvkm_wr32(device, 0x610b2c, 0x00000040);
nvkm_mask(device, 0x6104e0, 0x00000010, 0x00000010);
- nvkm_wr32(device, 0x680000, 0x00000000);
+ nvkm_wr32(device, 0x680000, chan->suspend_put);
nvkm_wr32(device, 0x6104e0, 0x00000013);
return gv100_disp_core_idle(chan);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
index 55db9a22b4be..660310b27f9c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c
@@ -179,6 +179,8 @@ nv50_disp_core_fini(struct nv50_disp_chan *chan)
nvkm_error(subdev, "core fini: %08x\n",
nvkm_rd32(device, 0x610200));
}
+
+ chan->suspend_put = nvkm_rd32(device, 0x640000);
}
static int
@@ -198,7 +200,7 @@ nv50_disp_core_init(struct nv50_disp_chan *chan)
nvkm_wr32(device, 0x610208, 0x00010000);
nvkm_wr32(device, 0x61020c, 0x00000000);
nvkm_mask(device, 0x610200, 0x00000010, 0x00000010);
- nvkm_wr32(device, 0x640000, 0x00000000);
+ nvkm_wr32(device, 0x640000, chan->suspend_put);
nvkm_wr32(device, 0x610200, 0x01000013);
/* wait for it to go inactive */
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
index edf7dd0d931d..76425e8586da 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
@@ -53,6 +53,8 @@ gf119_disp_dmac_fini(struct nv50_disp_chan *chan)
nvkm_error(subdev, "ch %d fini: %08x\n", user,
nvkm_rd32(device, 0x610490 + (ctrl * 0x10)));
}
+
+ chan->suspend_put = nvkm_rd32(device, 0x640000 + (ctrl * 0x1000));
}
static int
@@ -68,7 +70,7 @@ gf119_disp_dmac_init(struct nv50_disp_chan *chan)
nvkm_wr32(device, 0x610498 + (ctrl * 0x0010), 0x00010000);
nvkm_wr32(device, 0x61049c + (ctrl * 0x0010), 0x00000001);
nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00000010, 0x00000010);
- nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000);
+ nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), chan->suspend_put);
nvkm_wr32(device, 0x610490 + (ctrl * 0x0010), 0x00000013);
/* wait for it to go inactive */
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c
index f21a433199aa..da258df268d7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c
@@ -38,7 +38,7 @@ gp102_disp_dmac_init(struct nv50_disp_chan *chan)
nvkm_wr32(device, 0x611498 + (ctrl * 0x0010), 0x00010000);
nvkm_wr32(device, 0x61149c + (ctrl * 0x0010), 0x00000001);
nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00000010, 0x00000010);
- nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000);
+ nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), chan->suspend_put);
nvkm_wr32(device, 0x610490 + (ctrl * 0x0010), 0x00000013);
/* wait for it to go inactive */
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c
index eac0e42da354..fdb624ac6b87 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c
@@ -50,10 +50,12 @@ void
gv100_disp_dmac_fini(struct nv50_disp_chan *chan)
{
struct nvkm_device *device = chan->disp->base.engine.subdev.device;
+ const u32 uoff = (chan->chid.ctrl - 1) * 0x1000;
const u32 coff = chan->chid.ctrl * 0x04;
nvkm_mask(device, 0x6104e0 + coff, 0x00000010, 0x00000000);
gv100_disp_dmac_idle(chan);
nvkm_mask(device, 0x6104e0 + coff, 0x00000002, 0x00000000);
+ chan->suspend_put = nvkm_rd32(device, 0x690000 + uoff);
}
int
@@ -71,7 +73,7 @@ gv100_disp_dmac_init(struct nv50_disp_chan *chan)
nvkm_wr32(device, 0x610b2c + poff, 0x00000040);
nvkm_mask(device, 0x6104e0 + coff, 0x00000010, 0x00000010);
- nvkm_wr32(device, 0x690000 + uoff, 0x00000000);
+ nvkm_wr32(device, 0x690000 + uoff, chan->suspend_put);
nvkm_wr32(device, 0x6104e0 + coff, 0x00000013);
return gv100_disp_dmac_idle(chan);
}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
index 9e8a9d7a9b68..d0a7da96d62b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c
@@ -94,6 +94,8 @@ nv50_disp_dmac_fini(struct nv50_disp_chan *chan)
nvkm_error(subdev, "ch %d fini timeout, %08x\n", user,
nvkm_rd32(device, 0x610200 + (ctrl * 0x10)));
}
+
+ chan->suspend_put = nvkm_rd32(device, 0x640000 + (ctrl * 0x1000));
}
static int
@@ -109,7 +111,7 @@ nv50_disp_dmac_init(struct nv50_disp_chan *chan)
nvkm_wr32(device, 0x610208 + (ctrl * 0x0010), 0x00010000);
nvkm_wr32(device, 0x61020c + (ctrl * 0x0010), ctrl);
nvkm_mask(device, 0x610200 + (ctrl * 0x0010), 0x00000010, 0x00000010);
- nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000);
+ nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), chan->suspend_put);
nvkm_wr32(device, 0x610200 + (ctrl * 0x0010), 0x00000013);
/* wait for it to go inactive */
--
2.29.2