Fix iterator bug in Nex()

entity/base_comp.go

@@ -1,27 +1,26 @@
 package entity
 
-import "github.com/bloeys/nmage/assert"
+import "github.com/bloeys/nmage/registry"
 
 var _ Comp = &BaseComp{}
 
 type BaseComp struct {
-	Entity *Entity
+	Handle registry.Handle
 }
 
-func (b *BaseComp) base() {
+func (b BaseComp) baseComp() {
 }
 
-func (b *BaseComp) Init(parent *Entity) {
-	assert.T(parent != nil, "Component was initialized with a nil parent. That is not allowed.")
-	b.Entity = parent
+func (b *BaseComp) Init(parentHandle registry.Handle) {
+	b.Handle = parentHandle
 }
 
-func (b *BaseComp) Name() string {
+func (b BaseComp) Name() string {
 	return "Base Component"
 }
 
-func (b *BaseComp) Update() {
+func (b BaseComp) Update() {
 }
 
-func (b *BaseComp) Destroy() {
+func (b BaseComp) Destroy() {
 }

entity/comp.go

@@ -1,27 +1,38 @@
 package entity
 
-import "github.com/bloeys/nmage/assert"
+import (
+	"github.com/bloeys/nmage/assert"
+	"github.com/bloeys/nmage/registry"
+)
 
 type Comp interface {
 	// This ensures that implementors of the Comp interface
 	// always embed BaseComp
-	base()
+	baseComp()
 
 	Name() string
-	Init(parent *Entity)
+	Init(parentHandle registry.Handle)
 	Update()
 	Destroy()
 }
 
-func AddComp[T Comp](e *Entity, c T) {
-
-	assert.T(!HasComp[T](e), "Entity with id '%v' already has component of type '%T'", e.ID, c)
-
-	e.Comps = append(e.Comps, c)
-	c.Init(e)
+func NewCompContainer() CompContainer {
+	return CompContainer{Comps: []Comp{}}
 }
 
-func HasComp[T Comp](e *Entity) bool {
+type CompContainer struct {
+	Comps []Comp
+}
+
+func AddComp[T Comp](entityHandle registry.Handle, cc *CompContainer, c T) {
+
+	assert.T(!HasComp[T](cc), "Entity with id '%v' already has component of type '%T'", entityHandle, c)
+
+	cc.Comps = append(cc.Comps, c)
+	c.Init(entityHandle)
+}
+
+func HasComp[T Comp](e *CompContainer) bool {
 
 	for i := 0; i < len(e.Comps); i++ {
 
@@ -34,7 +45,7 @@ func HasComp[T Comp](e *Entity) bool {
 	return false
 }
 
-func GetComp[T Comp](e *Entity) (out T) {
+func GetComp[T Comp](e *CompContainer) (out T) {
 
 	for i := 0; i < len(e.Comps); i++ {
 
@@ -48,7 +59,7 @@ func GetComp[T Comp](e *Entity) (out T) {
 }
 
 // DestroyComp calls Destroy on the component and then removes it from the entities component list
-func DestroyComp[T Comp](e *Entity) {
+func DestroyComp[T Comp](e *CompContainer) {
 
 	for i := 0; i < len(e.Comps); i++ {
 

entity/entity.go

@@ -1,49 +1,7 @@
 package entity
 
-type EntityFlag byte
+import "github.com/bloeys/nmage/registry"
 
-const (
-	EntityFlag_None  EntityFlag = 0
-	EntityFlag_Alive EntityFlag = 1 << (iota - 1)
-)
-
-const (
-	GenerationShiftBits = 64 - 8
-	FlagsShiftBits      = 64 - 16
-	IndexBitMask        = 0x00_00_FFFF_FFFF_FFFF
-)
-
-type EntityHandle uint64
-
-type Entity struct {
-
-	// Byte 1: Generation; Byte 2: Flags; Bytes 3-8: Index
-	ID    EntityHandle
-	Comps []Comp
-}
-
-func (e *Entity) HasFlag(ef EntityFlag) bool {
-	return GetFlags(e.ID)&ef > 0
-}
-
-func (e *Entity) UpdateAllComps() {
-	for i := 0; i < len(e.Comps); i++ {
-		e.Comps[i].Update()
-	}
-}
-
-func GetGeneration(id EntityHandle) byte {
-	return byte(id >> GenerationShiftBits)
-}
-
-func GetFlags(id EntityHandle) EntityFlag {
-	return EntityFlag(id >> FlagsShiftBits)
-}
-
-func GetIndex(id EntityHandle) uint64 {
-	return uint64(id & IndexBitMask)
-}
-
-func NewEntityId(generation byte, flags EntityFlag, index uint64) EntityHandle {
-	return EntityHandle(index | (uint64(generation) << GenerationShiftBits) | (uint64(flags) << FlagsShiftBits))
+type Entity interface {
+	GetHandle() registry.Handle
 }

entity/registry.go

@@ -1,109 +0,0 @@
-package entity
-
-import (
-	"github.com/bloeys/nmage/assert"
-)
-
-var (
-	// The number of slots required to be in the free list before the free list
-	// is used for creating new entries
-	FreeListUsageThreshold uint32 = 20
-)
-
-type freeListitem struct {
-	EntityIndex uint64
-	nextFree    *freeListitem
-}
-
-type Registry struct {
-	EntityCount uint64
-	Entities    []Entity
-
-	FreeList     *freeListitem
-	FreeListSize uint32
-}
-
-func (r *Registry) NewEntity() *Entity {
-
-	assert.T(r.EntityCount < uint64(len(r.Entities)), "Can not add more entities to registry because it is full")
-
-	entityToUseIndex := uint64(0)
-	var entityToUse *Entity = nil
-
-	if r.FreeList != nil && r.FreeListSize > FreeListUsageThreshold {
-
-		entityToUseIndex = r.FreeList.EntityIndex
-		entityToUse = &r.Entities[entityToUseIndex]
-		r.FreeList = r.FreeList.nextFree
-		r.FreeListSize--
-	} else {
-
-		for i := 0; i < len(r.Entities); i++ {
-
-			e := &r.Entities[i]
-			if e.HasFlag(EntityFlag_Alive) {
-				continue
-			}
-
-			entityToUse = e
-			entityToUseIndex = uint64(i)
-			break
-		}
-	}
-
-	if entityToUse == nil {
-		panic("failed to create new entity because we did not find a free spot in the registry. Why did the assert not go off?")
-	}
-
-	r.EntityCount++
-	entityToUse.ID = NewEntityId(GetGeneration(entityToUse.ID)+1, EntityFlag_Alive, entityToUseIndex)
-	assert.T(entityToUse.ID != 0, "Entity ID must not be zero")
-	return entityToUse
-}
-
-func (r *Registry) GetEntity(id EntityHandle) *Entity {
-
-	index := GetIndex(id)
-	gen := GetGeneration(id)
-
-	e := &r.Entities[index]
-	eGen := GetGeneration(e.ID)
-
-	if gen != eGen {
-		return nil
-	}
-
-	return e
-}
-
-// FreeEntity calls Destroy on all the entities components, resets the component list, resets the entity flags, then ads this entity to the free list
-func (r *Registry) FreeEntity(id EntityHandle) {
-
-	e := r.GetEntity(id)
-	if e == nil {
-		return
-	}
-
-	for i := 0; i < len(e.Comps); i++ {
-		e.Comps[i].Destroy()
-	}
-
-	r.EntityCount--
-	eIndex := GetIndex(e.ID)
-
-	e.Comps = []Comp{}
-	e.ID = NewEntityId(GetGeneration(e.ID), EntityFlag_None, eIndex)
-
-	r.FreeList = &freeListitem{
-		EntityIndex: eIndex,
-		nextFree:    r.FreeList,
-	}
-	r.FreeListSize++
-}
-
-func NewRegistry(size uint32) *Registry {
-	assert.T(size > 0, "Registry size must be more than zero")
-	return &Registry{
-		Entities: make([]Entity, size),
-	}
-}

level/level.go

@@ -2,19 +2,16 @@ package level
 
 import (
 	"github.com/bloeys/nmage/assert"
-	"github.com/bloeys/nmage/entity"
 )
 
 type Level struct {
-	*entity.Registry
 	Name string
 }
 
-func NewLevel(name string, maxEntities uint32) *Level {
+func NewLevel(name string) *Level {
 
 	assert.T(name != "", "Level name can not be empty")
 	return &Level{
-		Name:     name,
-		Registry: entity.NewRegistry(maxEntities),
+		Name: name,
 	}
 }

main.go

@@ -11,10 +11,10 @@ import (
 	"github.com/bloeys/nmage/engine"
 	"github.com/bloeys/nmage/entity"
 	"github.com/bloeys/nmage/input"
-	"github.com/bloeys/nmage/level"
 	"github.com/bloeys/nmage/logging"
 	"github.com/bloeys/nmage/materials"
 	"github.com/bloeys/nmage/meshes"
+	"github.com/bloeys/nmage/registry"
 	"github.com/bloeys/nmage/renderer/rend3dgl"
 	"github.com/bloeys/nmage/timing"
 	nmageimgui "github.com/bloeys/nmage/ui/imgui"
@@ -87,24 +87,39 @@ func (t *TransformComp) Name() string {
 
 func Test() {
 
-	lvl := level.NewLevel("test level", 1000)
-	e1 := lvl.Registry.NewEntity()
+	// lvl := level.NewLevel("test level")
+	testRegistry := registry.NewRegistry[int](100)
 
-	trComp := entity.GetComp[*TransformComp](e1)
+	e1, e1Handle := testRegistry.New()
+	e1CompContainer := entity.NewCompContainer()
+	fmt.Printf("Entity 1: %+v; Handle: %+v; Index: %+v; Gen: %+v; Flags: %+v\n", e1, e1Handle, e1Handle.Index(), e1Handle.Generation(), e1Handle.Flags())
+
+	trComp := entity.GetComp[*TransformComp](&e1CompContainer)
 	fmt.Println("Get comp before adding any:", trComp)
 
-	entity.AddComp(e1, &TransformComp{
+	entity.AddComp(e1Handle, &e1CompContainer, &TransformComp{
 		Pos:   gglm.NewVec3(0, 0, 0),
 		Rot:   gglm.NewQuatEulerXYZ(0, 0, 0),
 		Scale: gglm.NewVec3(0, 0, 0),
 	})
-	trComp = entity.GetComp[*TransformComp](e1)
+	trComp = entity.GetComp[*TransformComp](&e1CompContainer)
 	fmt.Println("Get transform comp:", trComp)
 
-	fmt.Printf("Entity: %+v\n", e1)
-	fmt.Printf("Entity: %+v\n", lvl.Registry.NewEntity())
-	fmt.Printf("Entity: %+v\n", lvl.Registry.NewEntity())
-	fmt.Printf("Entity: %+v\n", lvl.Registry.NewEntity())
+	e2, e2Handle := testRegistry.New()
+	e3, e3Handle := testRegistry.New()
+	e4, e4Handle := testRegistry.New()
+	fmt.Printf("Entity 2: %+v; Handle: %+v; Index: %+v; Gen: %+v; Flags: %+v\n", e2, e2Handle, e2Handle.Index(), e2Handle.Generation(), e2Handle.Flags())
+	fmt.Printf("Entity 3: %+v; Handle: %+v; Index: %+v; Gen: %+v; Flags: %+v\n", e3, e3Handle, e3Handle.Index(), e3Handle.Generation(), e3Handle.Flags())
+	fmt.Printf("Entity 4: %+v; Handle: %+v; Index: %+v; Gen: %+v; Flags: %+v\n", e4, e4Handle, e4Handle.Index(), e4Handle.Generation(), e4Handle.Flags())
+
+	*e2 = 1000
+	fmt.Printf("Entity 2 value after registry get: %+v\n", *testRegistry.Get(e2Handle))
+
+	testRegistry.Free(e2Handle)
+	fmt.Printf("Entity 2 value after free: %+v\n", testRegistry.Get(e2Handle))
+
+	e5, e5Handle := testRegistry.New()
+	fmt.Printf("Entity 5: %+v; Handle: %+v; Index: %+v; Gen: %+v; Flags: %+v\n", e5, e5Handle, e5Handle.Index(), e5Handle.Generation(), e5Handle.Flags())
 }
 
 func main() {
@@ -300,8 +315,6 @@ func (g *OurGame) Update() {
 	g.Win.SDLWin.SetTitle(fmt.Sprint("nMage (", timing.GetAvgFPS(), " fps)"))
 }
 
-var testString string
-
 func (g *OurGame) updateCameraLookAround() {
 
 	mouseX, mouseY := input.GetMouseMotion()

registry/iterator.go

@@ -0,0 +1,73 @@
+package registry
+
+// Iterator goes through the entire registry it was created from and
+// returns all alive items, and nil after its done.
+//
+// The iterator will still work if items are added/removed to the registry
+// after it was created, but the following conditions apply:
+//   - If items are removed, iterator will not show the removed items (assuming it didn't return them before their removal)
+//   - If items are added, the iterator will either only return older items (i.e. is not affected), or only return newer items (i.e. items that were going to be returned before will now not get returned in favor of newly inserted items), or a mix of old and new items.
+//     However, in all cases the iterator will *never* returns more items than were alive at the time of the iterator's creation.
+//   - If items were both added and removed, the iterator might follow either of the previous 2 cases or a combination of them
+//
+// To summarize: The iterator will *never* return more items than were alive at the time of its creation, and will *never* return freed items
+//
+// Example usage:
+//
+//	for item, handle := it.Next(); !it.IsDone(); item, handle = it.Next() {
+//		// Do stuff
+//	}
+type Iterator[T any] struct {
+	registry       *Registry[T]
+	remainingItems uint
+	currIndex      int
+}
+
+func (it *Iterator[T]) Next() (*T, Handle) {
+
+	if it.IsDone() {
+		return nil, 0
+	}
+
+	// If IsDone() only checked 'remainingItems', then when Next() returns the last item IsDone() will immediately be true which will cause loops to exit before processing the last item!
+	// However, with this check IsDone will remain false until Next() is called at least one more time after returning the last item which ensures the last item is processed in the loop.
+	//
+	// In cases where iterator is created on an empty registry, IsDone() will report true and the above check will return early
+	if it.remainingItems == 0 {
+		it.currIndex = -1
+		return nil, 0
+	}
+
+	for ; it.currIndex < len(it.registry.Handles); it.currIndex++ {
+
+		handle := it.registry.Handles[it.currIndex]
+		if !handle.HasFlag(HandleFlag_Alive) {
+			continue
+		}
+
+		item := &it.registry.Items[it.currIndex]
+		it.currIndex++
+		it.remainingItems--
+		return item, handle
+	}
+
+	// If we reached here means we iterated to the end and didn't find anything, which probably
+	// means that the registry changed since we were created, and that remainingItems is not accurate.
+	//
+	// As such, we zero remaining items so that this iterator is considered done
+	it.currIndex = -1
+	it.remainingItems = 0
+	return nil, 0
+}
+
+func (it *Iterator[T]) IsDone() bool {
+
+	if it.remainingItems != 0 {
+		return false
+	}
+
+	// We have two cases here:
+	// 1. Index of zero means Next() never returned an item. Remaining items of zero without returning anything means we have an empty registry and so its safe to report done
+	// 2. Negative index means Next() has detected we reached the end and that its safe to report being done
+	return it.currIndex <= 0
+}

registry/registry.go

@@ -0,0 +1,131 @@
+package registry
+
+import (
+	"math"
+
+	"github.com/bloeys/nmage/assert"
+)
+
+type freeListitem struct {
+	ItemIndex uint64
+	nextFree  *freeListitem
+}
+
+// Registry is a storage data structure that can efficiently create/get/free items using generational indices.
+// Each item stored in the registry is associated with a 'handle' object that is used to get and free objects
+//
+// The registry 'owns' all items it stores and returns pointers to items in its array. All items are allocated upfront.
+//
+// It is NOT safe to concurrently create or free items. However, it is SAFE to concurrently get items
+type Registry[T any] struct {
+	ItemCount uint
+	Handles   []Handle
+	Items     []T
+
+	FreeList     *freeListitem
+	FreeListSize uint32
+
+	// The number of slots required to be in the free list before the free list
+	// is used for creating new entries
+	FreeListUsageThreshold uint32
+}
+
+func (r *Registry[T]) New() (*T, Handle) {
+
+	assert.T(r.ItemCount < uint(len(r.Handles)), "Can not add more entities to registry because it is full")
+
+	var index uint64 = math.MaxUint64
+
+	// Find index to use for the new item
+	if r.FreeList != nil && r.FreeListSize > r.FreeListUsageThreshold {
+
+		index = r.FreeList.ItemIndex
+
+		r.FreeList = r.FreeList.nextFree
+		r.FreeListSize--
+	} else {
+
+		for i := 0; i < len(r.Handles); i++ {
+
+			handle := r.Handles[i]
+
+			if handle.HasFlag(HandleFlag_Alive) {
+				continue
+			}
+
+			index = uint64(i)
+			break
+		}
+	}
+
+	if index == math.MaxUint64 {
+		panic("failed to create new entity because we did not find a free spot in the registry. Why did the item count assert not go off?")
+	}
+
+	var newItem T
+	newHandle := NewHandle(r.Handles[index].Generation()+1, HandleFlag_Alive, index)
+	assert.T(newHandle != 0, "Entity handle must not be zero")
+
+	r.ItemCount++
+	r.Handles[index] = newHandle
+	r.Items[index] = newItem
+
+	// It is very important we return directly from the items array, because if we return
+	// a pointer to newItem, and T is a value not a pointer, then newItem and what's stored in items will be different
+	return &r.Items[index], newHandle
+}
+
+func (r *Registry[T]) Get(id Handle) *T {
+
+	index := id.Index()
+	assert.T(index < uint64(len(r.Handles)), "Failed to get entity because of invalid entity handle. Handle index is %d while registry only has %d slots. Handle: %+v", index, r.ItemCount, id)
+
+	handle := r.Handles[index]
+	if handle.Generation() != id.Generation() || !handle.HasFlag(HandleFlag_Alive) {
+		return nil
+	}
+
+	item := &r.Items[index]
+	return item
+}
+
+// Free resets the entity flags then adds this entity to the free list
+func (r *Registry[T]) Free(id Handle) {
+
+	index := id.Index()
+	assert.T(index < uint64(len(r.Handles)), "Failed to free entity because of invalid entity handle. Handle index is %d while registry only has %d slots. Handle: %+v", index, r.ItemCount, id)
+
+	// Nothing to do if already free
+	handle := r.Handles[index]
+	if handle.Generation() != id.Generation() || !handle.HasFlag(HandleFlag_Alive) {
+		return
+	}
+
+	// Generation is incremented on aquire, so here we just reset flags
+	r.ItemCount--
+	r.Handles[index] = NewHandle(id.Generation(), HandleFlag_None, index)
+
+	// Add to free list
+	r.FreeList = &freeListitem{
+		ItemIndex: index,
+		nextFree:  r.FreeList,
+	}
+	r.FreeListSize++
+}
+
+func (r *Registry[T]) NewIterator() Iterator[T] {
+	return Iterator[T]{
+		registry:       r,
+		remainingItems: r.ItemCount,
+		currIndex:      0,
+	}
+}
+
+func NewRegistry[T any](size uint32) *Registry[T] {
+	assert.T(size > 0, "Registry size must be more than zero")
+	return &Registry[T]{
+		Handles:                make([]Handle, size),
+		Items:                  make([]T, size),
+		FreeListUsageThreshold: 30,
+	}
+}

registry/registry_handle.go

@@ -0,0 +1,37 @@
+package registry
+
+type HandleFlag byte
+
+const (
+	HandleFlag_None  HandleFlag = 0
+	HandleFlag_Alive HandleFlag = 1 << (iota - 1)
+)
+
+const (
+	GenerationShiftBits = 64 - 8
+	FlagsShiftBits      = 64 - 16
+	IndexBitMask        = 0x00_00_FFFF_FFFF_FFFF
+)
+
+// Byte 1: Generation; Byte 2: Flags; Bytes 3-8: Index
+type Handle uint64
+
+func (h Handle) HasFlag(ef HandleFlag) bool {
+	return h.Flags()&ef > 0
+}
+
+func (h Handle) Generation() byte {
+	return byte(h >> GenerationShiftBits)
+}
+
+func (h Handle) Flags() HandleFlag {
+	return HandleFlag(h >> FlagsShiftBits)
+}
+
+func (h Handle) Index() uint64 {
+	return uint64(h & IndexBitMask)
+}
+
+func NewHandle(generation byte, flags HandleFlag, index uint64) Handle {
+	return Handle(index | (uint64(generation) << GenerationShiftBits) | (uint64(flags) << FlagsShiftBits))
+}