GameInfoCache: Keep properly track of what's already loaded, lots of cleanup #8961

Job	Run time
build (ubuntu-latest, test, clang, clang++, ./b.sh --headless --unittest, clang-normal)	5m 27s
build-uwp	7m 58s
build-windows (x64)	10m 8s
build (ubuntu-latest, gcc, g++, ./b.sh --headless --unittest, gcc-normal)	6m 2s
build-windows (ARM64)	10m 31s
build (ubuntu-latest, android, clang, clang++, cd android && ./ab.sh -j2 APP_ABI=arm64-v8a UNITTE...	11m 27s
build (ubuntu-latest, android, clang, clang++, cd android && ./ab.sh -j2 APP_ABI=armeabi-v7a UNIT...	7m 16s
build (ubuntu-latest, android, clang, clang++, cd android && ./ab.sh -j2 APP_ABI=x86_64 UNITTEST=...	10m 55s
build (ubuntu-latest, android, clang, clang++, cd android && ./ab.sh -j2 APP_ABI=arm64-v8a OPENXR...	10m 19s
build (ubuntu-latest, android, clang, clang++, ./b.sh --libretro_android ppsspp_libretro, android...	6m 45s
build (ubuntu-latest, qt, gcc, g++, ./b.sh --qt, qt)	5m 33s
build (ubuntu-latest, libretro, gcc, g++, make -C libretro -f Makefile -j2, gcc-libretro)	4m 55s
build (ubuntu-latest, libretro, clang, clang++, make -C libretro -f Makefile -j2, clang-libretro)	4m 54s
build (macos-latest, test, clang, clang++, ./b.sh --headless --unittest --fat --no-png --no-sdl2,...	15m 18s
build (macos-latest, ios, clang, clang++, ./b.sh --ios, ios)	18m 48s
test-windows	45s
test (ubuntu-latest)	1m 2s
test (macos-latest)	48s
	2h 18m 51s

Provide feedback