drm/nouveau/fifo: add new channel lookup interfaces

- supports per-runlist CHIDs
- channel group lock held across reference, rather than global lock

v2:
- remove unnecessary parenthesis

Signed-off-by: Ben Skeggs <bskeggs@redhat.com>
Reviewed-by: Lyude Paul <lyude@redhat.com>

7 files changed, 113 insertions, 74 deletions

diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
index e51fb10a04f7..83da63bdf3c4 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/base.c
@@ -63,63 +63,6 @@ nvkm_fifo_fault(struct nvkm_fifo *fifo, struct nvkm_fault_data *info)
 }
 
 void
-nvkm_fifo_chan_put(struct nvkm_fifo *fifo, unsigned long flags,
-		   struct nvkm_fifo_chan **pchan)
-{
-	struct nvkm_fifo_chan *chan = *pchan;
-	if (likely(chan)) {
-		*pchan = NULL;
-		spin_unlock_irqrestore(&fifo->lock, flags);
-	}
-}
-
-struct nvkm_fifo_chan *
-nvkm_fifo_chan_inst_locked(struct nvkm_fifo *fifo, u64 inst)
-{
-	struct nvkm_fifo_chan *chan;
-	list_for_each_entry(chan, &fifo->chan, head) {
-		if (chan->inst->addr == inst) {
-			list_del(&chan->head);
-			list_add(&chan->head, &fifo->chan);
-			return chan;
-		}
-	}
-	return NULL;
-}
-
-struct nvkm_fifo_chan *
-nvkm_fifo_chan_inst(struct nvkm_fifo *fifo, u64 inst, unsigned long *rflags)
-{
-	struct nvkm_fifo_chan *chan;
-	unsigned long flags;
-	spin_lock_irqsave(&fifo->lock, flags);
-	if ((chan = nvkm_fifo_chan_inst_locked(fifo, inst))) {
-		*rflags = flags;
-		return chan;
-	}
-	spin_unlock_irqrestore(&fifo->lock, flags);
-	return NULL;
-}
-
-struct nvkm_fifo_chan *
-nvkm_fifo_chan_chid(struct nvkm_fifo *fifo, int chid, unsigned long *rflags)
-{
-	struct nvkm_fifo_chan *chan;
-	unsigned long flags;
-	spin_lock_irqsave(&fifo->lock, flags);
-	list_for_each_entry(chan, &fifo->chan, head) {
-		if (chan->chid == chid) {
-			list_del(&chan->head);
-			list_add(&chan->head, &fifo->chan);
-			*rflags = flags;
-			return chan;
-		}
-	}
-	spin_unlock_irqrestore(&fifo->lock, flags);
-	return NULL;
-}
-
-void
 nvkm_fifo_kevent(struct nvkm_fifo *fifo, int chid)
 {
 	nvkm_event_ntfy(&fifo->kevent, chid, NVKM_FIFO_EVENT_KILLED);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
index ea8148bad45e..bfbd5ec9a720 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/chan.c
@@ -302,6 +302,56 @@ nvkm_fifo_chan_dtor(struct nvkm_object *object)
 	return data;
 }
 
+void
+nvkm_chan_put(struct nvkm_chan **pchan, unsigned long irqflags)
+{
+	struct nvkm_chan *chan = *pchan;
+
+	if (!chan)
+		return;
+
+	*pchan = NULL;
+	spin_unlock_irqrestore(&chan->cgrp->lock, irqflags);
+}
+
+struct nvkm_chan *
+nvkm_chan_get_inst(struct nvkm_engine *engine, u64 inst, unsigned long *pirqflags)
+{
+	struct nvkm_fifo *fifo = engine->subdev.device->fifo;
+	struct nvkm_runl *runl;
+	struct nvkm_engn *engn;
+	struct nvkm_chan *chan;
+
+	nvkm_runl_foreach(runl, fifo) {
+		nvkm_runl_foreach_engn(engn, runl) {
+			if (engine == &fifo->engine || engn->engine == engine) {
+				chan = nvkm_runl_chan_get_inst(runl, inst, pirqflags);
+				if (chan || engn->engine == engine)
+					return chan;
+			}
+		}
+	}
+
+	return NULL;
+}
+
+struct nvkm_chan *
+nvkm_chan_get_chid(struct nvkm_engine *engine, int id, unsigned long *pirqflags)
+{
+	struct nvkm_fifo *fifo = engine->subdev.device->fifo;
+	struct nvkm_runl *runl;
+	struct nvkm_engn *engn;
+
+	nvkm_runl_foreach(runl, fifo) {
+		nvkm_runl_foreach_engn(engn, runl) {
+			if (fifo->chid || engn->engine == engine)
+				return nvkm_runl_chan_get_chid(runl, id, pirqflags);
+		}
+	}
+
+	return NULL;
+}
+
 static const struct nvkm_object_func
 nvkm_fifo_chan_func = {
 	.dtor = nvkm_fifo_chan_dtor,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
index 7fc4c37b407f..7c1db7721ff6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gf100.c
@@ -84,13 +84,13 @@ gf100_runq_intr(struct nvkm_runq *runq, struct nvkm_runl *null)
 
 	if (show) {
 		nvkm_snprintbf(msg, sizeof(msg), runq->func->intr_0_names, show);
-		chan = nvkm_fifo_chan_chid(runq->fifo, chid, &flags);
+		chan = nvkm_chan_get_chid(&runq->fifo->engine, chid, &flags);
 		nvkm_error(subdev, "PBDMA%d: %08x [%s] ch %d [%010llx %s] "
 				   "subc %d mthd %04x data %08x\n",
 			   runq->id, show, msg, chid, chan ? chan->inst->addr : 0,
 			   chan ? chan->object.client->name : "unknown",
 			   subc, mthd, data);
-		nvkm_fifo_chan_put(runq->fifo, flags, &chan);
+		nvkm_chan_put(&chan, flags);
 	}
 
 	nvkm_wr32(device, 0x0400c0 + (runq->id * 0x2000), 0x80600008);
@@ -367,9 +367,9 @@ gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *inf
 	struct nvkm_device *device = subdev->device;
 	const struct nvkm_enum *er, *ee, *ec, *ea;
 	struct nvkm_engine *engine = NULL;
-	struct nvkm_fifo_chan *chan;
 	struct nvkm_runl *runl;
 	struct nvkm_engn *engn;
+	struct nvkm_chan *chan;
 	unsigned long flags;
 	char ct[8] = "HUB/";
 
@@ -409,7 +409,7 @@ gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *inf
 		}
 	}
 
-	chan = nvkm_fifo_chan_inst(fifo, info->inst, &flags);
+	chan = nvkm_chan_get_inst(&fifo->engine, info->inst, &flags);
 
 	nvkm_error(subdev,
 		   "fault %02x [%s] at %016llx engine %02x [%s] client %02x "
@@ -427,7 +427,7 @@ gf100_fifo_mmu_fault_recover(struct nvkm_fifo *fifo, struct nvkm_fault_data *inf
 	if (engine && chan)
 		gf100_fifo_recover(gf100_fifo(fifo), engine, (void *)chan);
 
-	nvkm_fifo_chan_put(fifo, flags, &chan);
+	nvkm_chan_put(&chan, flags);
 }
 
 static const struct nvkm_fifo_func_mmu_fault
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
index b899127cfd72..85f22fce11ed 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/nv04.c
@@ -30,7 +30,6 @@
 #include "channv04.h"
 #include "regsnv04.h"
 
-#include <core/client.h>
 #include <core/ramht.h>
 #include <subdev/instmem.h>
 #include <subdev/mc.h>
@@ -241,7 +240,7 @@ nv04_fifo_intr_cache_error(struct nvkm_fifo *fifo, u32 chid, u32 get)
 {
 	struct nvkm_subdev *subdev = &fifo->engine.subdev;
 	struct nvkm_device *device = subdev->device;
-	struct nvkm_fifo_chan *chan;
+	struct nvkm_chan *chan;
 	unsigned long flags;
 	u32 pull0 = nvkm_rd32(device, 0x003250);
 	u32 mthd, data;
@@ -264,12 +263,12 @@ nv04_fifo_intr_cache_error(struct nvkm_fifo *fifo, u32 chid, u32 get)
 
 	if (!(pull0 & 0x00000100) ||
 	    !nv04_fifo_swmthd(device, chid, mthd, data)) {
-		chan = nvkm_fifo_chan_chid(fifo, chid, &flags);
+		chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags);
 		nvkm_error(subdev, "CACHE_ERROR - "
 			   "ch %d [%s] subc %d mthd %04x data %08x\n",
-			   chid, chan ? chan->object.client->name : "unknown",
+			   chid, chan ? chan->name : "unknown",
 			   (mthd >> 13) & 7, mthd & 0x1ffc, data);
-		nvkm_fifo_chan_put(fifo, flags, &chan);
+		nvkm_chan_put(&chan, flags);
 	}
 
 	nvkm_wr32(device, NV04_PFIFO_CACHE1_DMA_PUSH, 0);
@@ -296,12 +295,12 @@ nv04_fifo_intr_dma_pusher(struct nvkm_fifo *fifo, u32 chid)
 	u32 dma_put = nvkm_rd32(device, 0x003240);
 	u32 push = nvkm_rd32(device, 0x003220);
 	u32 state = nvkm_rd32(device, 0x003228);
-	struct nvkm_fifo_chan *chan;
+	struct nvkm_chan *chan;
 	unsigned long flags;
 	const char *name;
 
-	chan = nvkm_fifo_chan_chid(fifo, chid, &flags);
-	name = chan ? chan->object.client->name : "unknown";
+	chan = nvkm_chan_get_chid(&fifo->engine, chid, &flags);
+	name = chan ? chan->name : "unknown";
 	if (device->card_type == NV_50) {
 		u32 ho_get = nvkm_rd32(device, 0x003328);
 		u32 ho_put = nvkm_rd32(device, 0x003320);
@@ -332,7 +331,7 @@ nv04_fifo_intr_dma_pusher(struct nvkm_fifo *fifo, u32 chid)
 		if (dma_get != dma_put)
 			nvkm_wr32(device, 0x003244, dma_put);
 	}
-	nvkm_fifo_chan_put(fifo, flags, &chan);
+	nvkm_chan_put(&chan, flags);
 
 	nvkm_wr32(device, 0x003228, 0x00000000);
 	nvkm_wr32(device, 0x003220, 0x00000001);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
index 19c6e706f33e..7bc7b89ffd18 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/priv.h
@@ -14,9 +14,6 @@ struct gk104_fifo_chan;
 void nvkm_fifo_kevent(struct nvkm_fifo *, int chid);
 void nvkm_fifo_recover_chan(struct nvkm_fifo *, int chid);
 
-struct nvkm_fifo_chan *
-nvkm_fifo_chan_inst_locked(struct nvkm_fifo *, u64 inst);
-
 struct nvkm_fifo_chan_oclass;
 struct nvkm_fifo_func {
 	void *(*dtor)(struct nvkm_fifo *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c
index 0b75565bfbc1..e4984e1e7c65 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.c
@@ -20,12 +20,59 @@
  * OTHER DEALINGS IN THE SOFTWARE.
  */
 #include "runl.h"
+#include "cgrp.h"
 #include "chan.h"
 #include "chid.h"
 #include "priv.h"
 
+#include <core/gpuobj.h>
 #include <subdev/top.h>
 
+struct nvkm_chan *
+nvkm_runl_chan_get_inst(struct nvkm_runl *runl, u64 inst, unsigned long *pirqflags)
+{
+	struct nvkm_chid *chid = runl->chid;
+	struct nvkm_chan *chan;
+	unsigned long flags;
+	int id;
+
+	spin_lock_irqsave(&chid->lock, flags);
+	for_each_set_bit(id, chid->used, chid->nr) {
+		chan = chid->data[id];
+		if (likely(chan)) {
+			if (chan->inst->addr == inst) {
+				spin_lock(&chan->cgrp->lock);
+				*pirqflags = flags;
+				spin_unlock(&chid->lock);
+				return chan;
+			}
+		}
+	}
+	spin_unlock_irqrestore(&chid->lock, flags);
+	return NULL;
+}
+
+struct nvkm_chan *
+nvkm_runl_chan_get_chid(struct nvkm_runl *runl, int id, unsigned long *pirqflags)
+{
+	struct nvkm_chid *chid = runl->chid;
+	struct nvkm_chan *chan;
+	unsigned long flags;
+
+	spin_lock_irqsave(&chid->lock, flags);
+	if (!WARN_ON(id >= chid->nr)) {
+		chan = chid->data[id];
+		if (likely(chan)) {
+			spin_lock(&chan->cgrp->lock);
+			*pirqflags = flags;
+			spin_unlock(&chid->lock);
+			return chan;
+		}
+	}
+	spin_unlock_irqrestore(&chid->lock, flags);
+	return NULL;
+}
+
 void
 nvkm_runl_del(struct nvkm_runl *runl)
 {
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h
index b223da2d4399..e618e326b3c3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/runl.h
@@ -51,6 +51,9 @@ struct nvkm_engn *nvkm_runl_add(struct nvkm_runl *, int engi, const struct nvkm_
 				enum nvkm_subdev_type, int inst);
 void nvkm_runl_del(struct nvkm_runl *);
 
+struct nvkm_chan *nvkm_runl_chan_get_chid(struct nvkm_runl *, int chid, unsigned long *irqflags);
+struct nvkm_chan *nvkm_runl_chan_get_inst(struct nvkm_runl *, u64 inst, unsigned long *irqflags);
+
 #define nvkm_runl_find_engn(engn,runl,cond) nvkm_list_find(engn, &(runl)->engns, head, (cond))
 
 #define nvkm_runl_foreach(runl,fifo) list_for_each_entry((runl), &(fifo)->runls, head)