Fixed a bug with readback_resolve and readback_memexport that was responsible for a large portion of their overhead. readback_memexport and resolve are now usable for games, depending on your hardware. in my case games that were slideshows now run at like 20-30 fps, and my hardware isnt the best for xenia.
add split_map class for mapping keys to values in a way that optimizes for frequent searches and infrequent insertions/removals remove jump table implementation of GetColorRenderTargetFormatComponentCount, it was appearing relatively high in profiles. instead pack the component counts into a single 32 bit word, which is indexed by shifting Add cvar to align all basic blocks to a boundary Add mmio aware load paths liberally apply XE_RESTRICT in ringbuffer related code Removed the IS_TRUE and IS_FALSE opcodes, they were pointless duplicates of COMPARE_EQ/COMPARE_NE and i want to simplify our set of opcodes for future backends More work on LVSR/LVSL/STVR/STVL opcodes Optimized X64 translated code emission, now only compute instrkey once Add code for pre-computing integer division magic numbers Optimized GetHostViewportInfo a little Move args for GetHostViewportInfo into a class, cache the result and compare for future queries. moved GetHostViewportInfo far lower on the profile Add (currently not functional, and very racy) asynchronous memcpy code. will improve it and actually use it in future commits. Add non-temporal memcpy function for huge page-aligned allocations. Used for copying to shared memory/readback hoist are_accumulated_render_targets_valid_ check out of loop in render_target_cache already bound check. Add stosb/movsb code for small constant memcpys/memsets that arent worth the overhead of memcpy/memset
This commit is contained in:
chss95cs@gmail.com
415
src/xenia/base/dma.cc
Normal file
415
src/xenia/base/dma.cc
Normal file
@@ -0,0 +1,415 @@
|
||||
#include "dma.h"
|
||||
|
||||
template <size_t N, typename... Ts>
|
||||
static void xedmaloghelper(const char (&fmt)[N], Ts... args) {
|
||||
char buffer[1024];
|
||||
sprintf_s(buffer, fmt, args...);
|
||||
XELOGI("%s", buffer);
|
||||
}
|
||||
|
||||
//#define XEDMALOG(...) XELOGI("XeDma: " __VA_ARGS__)
|
||||
//#define XEDMALOG(...) xedmaloghelper("XeDma: " __VA_ARGS__)
|
||||
#define XEDMALOG(...) static_cast<void>(0)
|
||||
using xe::swcache::CacheLine;
|
||||
static constexpr unsigned NUM_CACHELINES_IN_PAGE = 4096 / sizeof(CacheLine);
|
||||
|
||||
XE_FORCEINLINE
|
||||
static void XeCopy16384Streaming(CacheLine* XE_RESTRICT to,
|
||||
CacheLine* XE_RESTRICT from) {
|
||||
uint32_t num_lines_for_8k = 4096 / XE_HOST_CACHE_LINE_SIZE;
|
||||
|
||||
CacheLine* dest1 = to;
|
||||
CacheLine* src1 = from;
|
||||
|
||||
CacheLine* dest2 = to + NUM_CACHELINES_IN_PAGE;
|
||||
CacheLine* src2 = from + NUM_CACHELINES_IN_PAGE;
|
||||
|
||||
CacheLine* dest3 = to + (NUM_CACHELINES_IN_PAGE * 2);
|
||||
CacheLine* src3 = from + (NUM_CACHELINES_IN_PAGE * 2);
|
||||
|
||||
CacheLine* dest4 = to + (NUM_CACHELINES_IN_PAGE * 3);
|
||||
CacheLine* src4 = from + (NUM_CACHELINES_IN_PAGE * 3);
|
||||
#pragma loop(no_vector)
|
||||
for (uint32_t i = 0; i < num_lines_for_8k; ++i) {
|
||||
xe::swcache::CacheLine line0, line1, line2, line3;
|
||||
|
||||
xe::swcache::ReadLine(&line0, src1 + i);
|
||||
xe::swcache::ReadLine(&line1, src2 + i);
|
||||
xe::swcache::ReadLine(&line2, src3 + i);
|
||||
xe::swcache::ReadLine(&line3, src4 + i);
|
||||
XE_MSVC_REORDER_BARRIER();
|
||||
xe::swcache::WriteLineNT(dest1 + i, &line0);
|
||||
xe::swcache::WriteLineNT(dest2 + i, &line1);
|
||||
|
||||
xe::swcache::WriteLineNT(dest3 + i, &line2);
|
||||
xe::swcache::WriteLineNT(dest4 + i, &line3);
|
||||
}
|
||||
XE_MSVC_REORDER_BARRIER();
|
||||
}
|
||||
|
||||
namespace xe::dma {
|
||||
XE_FORCEINLINE
|
||||
static void vastcpy_impl(CacheLine* XE_RESTRICT physaddr,
|
||||
CacheLine* XE_RESTRICT rdmapping,
|
||||
uint32_t written_length) {
|
||||
static constexpr unsigned NUM_LINES_FOR_16K = 16384 / XE_HOST_CACHE_LINE_SIZE;
|
||||
|
||||
while (written_length >= 16384) {
|
||||
XeCopy16384Streaming(physaddr, rdmapping);
|
||||
|
||||
physaddr += NUM_LINES_FOR_16K;
|
||||
rdmapping += NUM_LINES_FOR_16K;
|
||||
|
||||
written_length -= 16384;
|
||||
}
|
||||
|
||||
if (!written_length) {
|
||||
return;
|
||||
}
|
||||
uint32_t num_written_lines = written_length / XE_HOST_CACHE_LINE_SIZE;
|
||||
|
||||
uint32_t i = 0;
|
||||
|
||||
for (; i + 1 < num_written_lines; i += 2) {
|
||||
xe::swcache::CacheLine line0, line1;
|
||||
|
||||
xe::swcache::ReadLine(&line0, rdmapping + i);
|
||||
|
||||
xe::swcache::ReadLine(&line1, rdmapping + i + 1);
|
||||
XE_MSVC_REORDER_BARRIER();
|
||||
xe::swcache::WriteLineNT(physaddr + i, &line0);
|
||||
xe::swcache::WriteLineNT(physaddr + i + 1, &line1);
|
||||
}
|
||||
|
||||
if (i < num_written_lines) {
|
||||
xe::swcache::CacheLine line0;
|
||||
|
||||
xe::swcache::ReadLine(&line0, rdmapping + i);
|
||||
xe::swcache::WriteLineNT(physaddr + i, &line0);
|
||||
}
|
||||
}
|
||||
|
||||
XE_NOINLINE
|
||||
void vastcpy(uint8_t* XE_RESTRICT physaddr, uint8_t* XE_RESTRICT rdmapping,
|
||||
uint32_t written_length) {
|
||||
return vastcpy_impl((CacheLine*)physaddr, (CacheLine*)rdmapping,
|
||||
written_length);
|
||||
}
|
||||
|
||||
#define XEDMA_NUM_WORKERS 4
|
||||
class alignas(256) XeDMACGeneric : public XeDMAC {
|
||||
struct alignas(XE_HOST_CACHE_LINE_SIZE) {
|
||||
std::atomic<uint64_t> free_job_slots_;
|
||||
std::atomic<uint64_t> jobs_submitted_;
|
||||
std::atomic<uint64_t> jobs_completed_;
|
||||
std::atomic<uint32_t> num_workers_awoken_;
|
||||
std::atomic<uint32_t> current_job_serial_;
|
||||
|
||||
} dma_volatile_;
|
||||
|
||||
alignas(XE_HOST_CACHE_LINE_SIZE) XeDMAJob jobs_[64];
|
||||
|
||||
volatile uint32_t jobserials_[64];
|
||||
|
||||
alignas(XE_HOST_CACHE_LINE_SIZE)
|
||||
std::unique_ptr<threading::Event> job_done_signals_[64];
|
||||
// really dont like using unique pointer for this...
|
||||
std::unique_ptr<threading::Event> job_submitted_signal_;
|
||||
std::unique_ptr<threading::Event> job_completed_signal_;
|
||||
|
||||
std::unique_ptr<threading::Thread> scheduler_thread_;
|
||||
struct WorkSlice {
|
||||
uint8_t* destination;
|
||||
uint8_t* source;
|
||||
size_t numbytes;
|
||||
};
|
||||
std::unique_ptr<threading::Thread> workers_[XEDMA_NUM_WORKERS];
|
||||
std::unique_ptr<threading::Event> worker_has_work_; //[XEDMA_NUM_WORKERS];
|
||||
std::unique_ptr<threading::Event> worker_has_finished_[XEDMA_NUM_WORKERS];
|
||||
|
||||
threading::WaitHandle* worker_has_finished_nosafeptr_[XEDMA_NUM_WORKERS];
|
||||
WorkSlice worker_workslice_[XEDMA_NUM_WORKERS];
|
||||
|
||||
// chrispy: this is bad
|
||||
static uint32_t find_free_hole_in_dword(uint64_t dw) {
|
||||
XEDMALOG("Finding free hole in 0x%llX", dw);
|
||||
|
||||
for (uint32_t i = 0; i < 64; ++i) {
|
||||
if (dw & (1ULL << i)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
return i;
|
||||
}
|
||||
return ~0U;
|
||||
}
|
||||
|
||||
uint32_t allocate_free_dma_slot() {
|
||||
XEDMALOG("Allocating free slot");
|
||||
uint32_t got_slot = 0;
|
||||
uint64_t slots;
|
||||
uint64_t allocated_slot;
|
||||
|
||||
do {
|
||||
slots = dma_volatile_.free_job_slots_.load();
|
||||
|
||||
got_slot = find_free_hole_in_dword(slots);
|
||||
if (!~got_slot) {
|
||||
XEDMALOG("Didn't get a slot!");
|
||||
return ~0U;
|
||||
}
|
||||
allocated_slot = slots | (1ULL << got_slot);
|
||||
|
||||
} while (XE_UNLIKELY(!dma_volatile_.free_job_slots_.compare_exchange_strong(
|
||||
slots, allocated_slot)));
|
||||
XEDMALOG("Allocated slot %d", got_slot);
|
||||
return got_slot;
|
||||
}
|
||||
// chrispy: on x86 this can just be interlockedbittestandreset...
|
||||
void free_dma_slot(uint32_t slot) {
|
||||
XEDMALOG("Freeing slot %d", slot);
|
||||
uint64_t slots;
|
||||
|
||||
uint64_t deallocated_slot;
|
||||
|
||||
do {
|
||||
slots = dma_volatile_.free_job_slots_.load();
|
||||
|
||||
deallocated_slot = slots & (~(1ULL << slot));
|
||||
|
||||
} while (XE_UNLIKELY(!dma_volatile_.free_job_slots_.compare_exchange_strong(
|
||||
slots, deallocated_slot)));
|
||||
}
|
||||
|
||||
void DoDMAJob(uint32_t idx) {
|
||||
XeDMAJob& job = jobs_[idx];
|
||||
if (job.precall) {
|
||||
job.precall(&job);
|
||||
}
|
||||
// memcpy(job.destination, job.source, job.size);
|
||||
|
||||
size_t job_size = job.size;
|
||||
|
||||
size_t job_num_lines = job_size / XE_HOST_CACHE_LINE_SIZE;
|
||||
|
||||
size_t line_rounded = job_num_lines * XE_HOST_CACHE_LINE_SIZE;
|
||||
|
||||
size_t rem = job_size - line_rounded;
|
||||
|
||||
size_t num_per_worker = line_rounded / XEDMA_NUM_WORKERS;
|
||||
|
||||
XEDMALOG(
|
||||
"Distributing %d bytes from %p to %p across %d workers, remainder is "
|
||||
"%d",
|
||||
line_rounded, job.source, job.destination, XEDMA_NUM_WORKERS, rem);
|
||||
if (num_per_worker < 2048) {
|
||||
XEDMALOG("not distributing across workers, num_per_worker < 8192");
|
||||
// not worth splitting up
|
||||
memcpy(job.destination, job.source, job.size);
|
||||
job.signal_on_done->Set();
|
||||
} else {
|
||||
for (uint32_t i = 0; i < XEDMA_NUM_WORKERS; ++i) {
|
||||
worker_workslice_[i].destination =
|
||||
(i * num_per_worker) + job.destination;
|
||||
worker_workslice_[i].source = (i * num_per_worker) + job.source;
|
||||
|
||||
worker_workslice_[i].numbytes = num_per_worker;
|
||||
}
|
||||
if (rem) {
|
||||
__movsb(job.destination + line_rounded, job.source + line_rounded, rem);
|
||||
}
|
||||
// wake them up
|
||||
worker_has_work_->Set();
|
||||
XEDMALOG("Starting waitall for job");
|
||||
threading::WaitAll(worker_has_finished_nosafeptr_, XEDMA_NUM_WORKERS,
|
||||
false);
|
||||
|
||||
XEDMALOG("Waitall for job completed!");
|
||||
job.signal_on_done->Set();
|
||||
}
|
||||
if (job.postcall) {
|
||||
job.postcall(&job);
|
||||
}
|
||||
++dma_volatile_.jobs_completed_;
|
||||
}
|
||||
|
||||
void WorkerIter(uint32_t worker_index) {
|
||||
xenia_assert(worker_index < XEDMA_NUM_WORKERS);
|
||||
auto [dest, src, size] = worker_workslice_[worker_index];
|
||||
|
||||
// if (++dma_volatile_.num_workers_awoken_ == XEDMA_NUM_WORKERS ) {
|
||||
worker_has_work_->Reset();
|
||||
//}
|
||||
xenia_assert(size < (1ULL << 32));
|
||||
// memcpy(dest, src, size);
|
||||
dma::vastcpy(dest, src, static_cast<uint32_t>(size));
|
||||
}
|
||||
XE_NOINLINE
|
||||
void WorkerMainLoop(uint32_t worker_index) {
|
||||
do {
|
||||
XEDMALOG("Worker iter for worker %d", worker_index);
|
||||
WorkerIter(worker_index);
|
||||
|
||||
XEDMALOG("Worker %d is done\n", worker_index);
|
||||
threading::SignalAndWait(worker_has_finished_[worker_index].get(),
|
||||
worker_has_work_.get(), false);
|
||||
} while (true);
|
||||
}
|
||||
void WorkerMain(uint32_t worker_index) {
|
||||
XEDMALOG("Entered worker main loop, index %d", worker_index);
|
||||
threading::Wait(worker_has_work_.get(), false);
|
||||
XEDMALOG("First wait for worker %d completed, first job ever",
|
||||
worker_index);
|
||||
WorkerMainLoop(worker_index);
|
||||
}
|
||||
|
||||
static void WorkerMainForwarder(void* ptr) {
|
||||
// we aligned XeDma to 256 bytes and encode extra info in the low 8
|
||||
uintptr_t uptr = (uintptr_t)ptr;
|
||||
|
||||
uint32_t worker_index = (uint8_t)uptr;
|
||||
|
||||
uptr &= ~0xFFULL;
|
||||
|
||||
char name_buffer[64];
|
||||
sprintf_s(name_buffer, "dma_worker_%d", worker_index);
|
||||
|
||||
xe::threading::set_name(name_buffer);
|
||||
|
||||
reinterpret_cast<XeDMACGeneric*>(uptr)->WorkerMain(worker_index);
|
||||
}
|
||||
|
||||
void DMAMain() {
|
||||
XEDMALOG("DmaMain");
|
||||
do {
|
||||
threading::Wait(job_submitted_signal_.get(), false);
|
||||
|
||||
auto slots = dma_volatile_.free_job_slots_.load();
|
||||
|
||||
for (uint32_t i = 0; i < 64; ++i) {
|
||||
if (slots & (1ULL << i)) {
|
||||
XEDMALOG("Got new job at index %d in DMAMain", i);
|
||||
DoDMAJob(i);
|
||||
|
||||
free_dma_slot(i);
|
||||
|
||||
job_completed_signal_->Set();
|
||||
// break;
|
||||
}
|
||||
}
|
||||
|
||||
} while (true);
|
||||
}
|
||||
|
||||
static void DMAMainForwarder(void* ud) {
|
||||
xe::threading::set_name("dma_main");
|
||||
reinterpret_cast<XeDMACGeneric*>(ud)->DMAMain();
|
||||
}
|
||||
|
||||
public:
|
||||
virtual DMACJobHandle PushDMAJob(XeDMAJob* job) override {
|
||||
XEDMALOG("New job, %p to %p with size %d", job->source, job->destination,
|
||||
job->size);
|
||||
uint32_t slot;
|
||||
do {
|
||||
slot = allocate_free_dma_slot();
|
||||
if (!~slot) {
|
||||
XEDMALOG(
|
||||
"Didn't get a free slot, waiting for a job to complete before "
|
||||
"resuming.");
|
||||
threading::Wait(job_completed_signal_.get(), false);
|
||||
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
|
||||
} while (true);
|
||||
jobs_[slot] = *job;
|
||||
|
||||
jobs_[slot].signal_on_done = job_done_signals_[slot].get();
|
||||
jobs_[slot].signal_on_done->Reset();
|
||||
XEDMALOG("Setting job submit signal, pushed into slot %d", slot);
|
||||
|
||||
uint32_t new_serial = dma_volatile_.current_job_serial_++;
|
||||
|
||||
jobserials_[slot] = new_serial;
|
||||
|
||||
++dma_volatile_.jobs_submitted_;
|
||||
job_submitted_signal_->Set();
|
||||
return (static_cast<uint64_t>(new_serial) << 32) |
|
||||
static_cast<uint64_t>(slot);
|
||||
|
||||
// return job_done_signals_[slot].get();
|
||||
}
|
||||
|
||||
bool AllJobsDone() {
|
||||
return dma_volatile_.jobs_completed_ == dma_volatile_.jobs_submitted_;
|
||||
}
|
||||
virtual void WaitJobDone(DMACJobHandle handle) override {
|
||||
uint32_t serial = static_cast<uint32_t>(handle >> 32);
|
||||
uint32_t jobid = static_cast<uint32_t>(handle);
|
||||
do {
|
||||
if (jobserials_[jobid] != serial) {
|
||||
return; // done, our slot was reused
|
||||
}
|
||||
|
||||
auto waitres = threading::Wait(job_done_signals_[jobid].get(), false,
|
||||
std::chrono::milliseconds{1});
|
||||
|
||||
if (waitres == threading::WaitResult::kTimeout) {
|
||||
continue;
|
||||
} else {
|
||||
return;
|
||||
}
|
||||
} while (true);
|
||||
}
|
||||
virtual void WaitForIdle() override {
|
||||
while (!AllJobsDone()) {
|
||||
threading::MaybeYield();
|
||||
}
|
||||
}
|
||||
XeDMACGeneric() {
|
||||
XEDMALOG("Constructing xedma at addr %p", this);
|
||||
dma_volatile_.free_job_slots_.store(0ULL);
|
||||
dma_volatile_.jobs_submitted_.store(0ULL);
|
||||
dma_volatile_.jobs_completed_.store(0ULL);
|
||||
dma_volatile_.current_job_serial_.store(
|
||||
1ULL); // so that a jobhandle is never 0
|
||||
std::memset(jobs_, 0, sizeof(jobs_));
|
||||
job_submitted_signal_ = threading::Event::CreateAutoResetEvent(false);
|
||||
job_completed_signal_ = threading::Event::CreateAutoResetEvent(false);
|
||||
worker_has_work_ = threading::Event::CreateManualResetEvent(false);
|
||||
threading::Thread::CreationParameters worker_params{};
|
||||
worker_params.create_suspended = false;
|
||||
worker_params.initial_priority = threading::ThreadPriority::kBelowNormal;
|
||||
worker_params.stack_size = 65536; // dont need much stack at all
|
||||
|
||||
for (uint32_t i = 0; i < 64; ++i) {
|
||||
job_done_signals_[i] = threading::Event::CreateManualResetEvent(false);
|
||||
}
|
||||
for (uint32_t i = 0; i < XEDMA_NUM_WORKERS; ++i) {
|
||||
// worker_has_work_[i] = threading::Event::CreateAutoResetEvent(false);
|
||||
worker_has_finished_[i] = threading::Event::CreateAutoResetEvent(false);
|
||||
worker_has_finished_nosafeptr_[i] = worker_has_finished_[i].get();
|
||||
|
||||
uintptr_t encoded = reinterpret_cast<uintptr_t>(this);
|
||||
xenia_assert(!(encoded & 0xFFULL));
|
||||
xenia_assert(i < 256);
|
||||
|
||||
encoded |= i;
|
||||
|
||||
workers_[i] = threading::Thread::Create(worker_params, [encoded]() {
|
||||
XeDMACGeneric::WorkerMainForwarder((void*)encoded);
|
||||
});
|
||||
}
|
||||
threading::Thread::CreationParameters scheduler_params{};
|
||||
scheduler_params.create_suspended = false;
|
||||
scheduler_params.initial_priority = threading::ThreadPriority::kBelowNormal;
|
||||
scheduler_params.stack_size = 65536;
|
||||
scheduler_thread_ = threading::Thread::Create(scheduler_params, [this]() {
|
||||
XeDMACGeneric::DMAMainForwarder((void*)this);
|
||||
});
|
||||
}
|
||||
};
|
||||
XeDMAC* CreateDMAC() { return new XeDMACGeneric(); }
|
||||
} // namespace xe::dma
|
||||
46
src/xenia/base/dma.h
Normal file
46
src/xenia/base/dma.h
Normal file
@@ -0,0 +1,46 @@
|
||||
/**
|
||||
******************************************************************************
|
||||
* Xenia : Xbox 360 Emulator Research Project *
|
||||
******************************************************************************
|
||||
* Copyright 2020 Ben Vanik. All rights reserved. *
|
||||
* Released under the BSD license - see LICENSE in the root for more details. *
|
||||
******************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef XENIA_BASE_DMA_H_
|
||||
#define XENIA_BASE_DMA_H_
|
||||
#include "memory.h"
|
||||
#include "threading.h"
|
||||
namespace xe::dma {
|
||||
struct XeDMAJob;
|
||||
using DmaPrecall = void (*)(XeDMAJob* job);
|
||||
using DmaPostcall = void (*)(XeDMAJob* job);
|
||||
struct XeDMAJob {
|
||||
threading::Event* signal_on_done;
|
||||
uint8_t* destination;
|
||||
uint8_t* source;
|
||||
size_t size;
|
||||
DmaPrecall precall;
|
||||
DmaPostcall postcall;
|
||||
void* userdata1;
|
||||
void* userdata2;
|
||||
};
|
||||
using DMACJobHandle = uint64_t;
|
||||
|
||||
class XeDMAC {
|
||||
public:
|
||||
virtual ~XeDMAC() {}
|
||||
virtual DMACJobHandle PushDMAJob(XeDMAJob* job) = 0;
|
||||
virtual void WaitJobDone(DMACJobHandle handle) = 0;
|
||||
virtual void WaitForIdle() = 0;
|
||||
};
|
||||
|
||||
XeDMAC* CreateDMAC();
|
||||
// must be divisible by cache line size
|
||||
XE_NOINLINE
|
||||
void vastcpy(uint8_t* XE_RESTRICT physaddr, uint8_t* XE_RESTRICT rdmapping,
|
||||
uint32_t written_length);
|
||||
|
||||
} // namespace xe::dma
|
||||
|
||||
#endif // XENIA_BASE_DMA_H_
|
||||
@@ -377,29 +377,45 @@ int64_t m128_i64(const __m128& v) {
|
||||
return m128_i64<N>(_mm_castps_pd(v));
|
||||
}
|
||||
/*
|
||||
|
||||
std::min/max float has handling for nans, where if either argument is nan the first argument is returned
|
||||
|
||||
minss/maxss are different, if either argument is nan the second operand to the instruction is returned
|
||||
this is problematic because we have no assurances from the compiler on the argument ordering
|
||||
std::min/max float has handling for nans, where if either argument is
|
||||
nan the first argument is returned
|
||||
|
||||
so only use in places where nan handling is not needed
|
||||
minss/maxss are different, if either argument is nan the second operand
|
||||
to the instruction is returned this is problematic because we have no
|
||||
assurances from the compiler on the argument ordering
|
||||
|
||||
so only use in places where nan handling is not needed
|
||||
*/
|
||||
static float xe_minf(float x, float y) {
|
||||
XE_FORCEINLINE
|
||||
static float ArchMin(float x, float y) {
|
||||
return _mm_cvtss_f32(_mm_min_ss(_mm_set_ss(x), _mm_set_ss(y)));
|
||||
}
|
||||
static float xe_maxf(float x, float y) {
|
||||
XE_FORCEINLINE
|
||||
static float ArchMax(float x, float y) {
|
||||
return _mm_cvtss_f32(_mm_max_ss(_mm_set_ss(x), _mm_set_ss(y)));
|
||||
}
|
||||
static float xe_rcpf(float den) {
|
||||
XE_FORCEINLINE
|
||||
static float ArchReciprocal(float den) {
|
||||
return _mm_cvtss_f32(_mm_rcp_ss(_mm_set_ss(den)));
|
||||
}
|
||||
|
||||
#else
|
||||
static float xe_minf(float x, float y) { return std::min<float>(x, y); }
|
||||
static float xe_maxf(float x, float y) { return std::max<float>(x, y); }
|
||||
static float xe_rcpf(float den) { return 1.0f / den; }
|
||||
static float ArchMin(float x, float y) { return std::min<float>(x, y); }
|
||||
static float ArchMax(float x, float y) { return std::max<float>(x, y); }
|
||||
static float ArchReciprocal(float den) { return 1.0f / den; }
|
||||
#endif
|
||||
XE_FORCEINLINE
|
||||
static float RefineReciprocal(float initial, float den) {
|
||||
float t0 = initial * den;
|
||||
float t1 = t0 * initial;
|
||||
float rcp2 = initial + initial;
|
||||
return rcp2 - t1;
|
||||
}
|
||||
XE_FORCEINLINE
|
||||
static float ArchReciprocalRefined(float den) {
|
||||
return RefineReciprocal(ArchReciprocal(den), den);
|
||||
}
|
||||
|
||||
// Similar to the C++ implementation of XMConvertFloatToHalf and
|
||||
// XMConvertHalfToFloat from DirectXMath 3.00 (pre-3.04, which switched from the
|
||||
@@ -494,7 +510,101 @@ inline T sat_sub(T a, T b) {
|
||||
}
|
||||
return T(result);
|
||||
}
|
||||
namespace divisors {
|
||||
union IDivExtraInfo {
|
||||
uint32_t value_;
|
||||
struct {
|
||||
uint32_t shift_ : 31;
|
||||
uint32_t add_ : 1;
|
||||
} info;
|
||||
};
|
||||
// returns magicnum multiplier
|
||||
static uint32_t PregenerateUint32Div(uint32_t _denom, uint32_t& out_extra) {
|
||||
IDivExtraInfo extra;
|
||||
|
||||
uint32_t d = _denom;
|
||||
int p;
|
||||
uint32_t nc, delta, q1, r1, q2, r2;
|
||||
struct {
|
||||
unsigned M;
|
||||
int a;
|
||||
int s;
|
||||
} magu;
|
||||
magu.a = 0;
|
||||
nc = -1 - ((uint32_t) - (int32_t)d) % d;
|
||||
p = 31;
|
||||
q1 = 0x80000000 / nc;
|
||||
r1 = 0x80000000 - q1 * nc;
|
||||
q2 = 0x7FFFFFFF / d;
|
||||
r2 = 0x7FFFFFFF - q2 * d;
|
||||
do {
|
||||
p += 1;
|
||||
if (r1 >= nc - r1) {
|
||||
q1 = 2 * q1 + 1;
|
||||
r1 = 2 * r1 - nc;
|
||||
} else {
|
||||
q1 = 2 * q1;
|
||||
r1 = 2 * r1;
|
||||
}
|
||||
if (r2 + 1 >= d - r2) {
|
||||
if (q2 >= 0x7FFFFFFF) {
|
||||
magu.a = 1;
|
||||
}
|
||||
q2 = 2 * q2 + 1;
|
||||
r2 = 2 * r2 + 1 - d;
|
||||
|
||||
} else {
|
||||
if (q2 >= 0x80000000U) {
|
||||
magu.a = 1;
|
||||
}
|
||||
q2 = 2 * q2;
|
||||
r2 = 2 * r2 + 1;
|
||||
}
|
||||
delta = d - 1 - r2;
|
||||
} while (p < 64 && (q1 < delta || r1 == 0));
|
||||
|
||||
extra.info.add_ = magu.a;
|
||||
extra.info.shift_ = p - 32;
|
||||
out_extra = extra.value_;
|
||||
return static_cast<uint64_t>(q2 + 1);
|
||||
}
|
||||
|
||||
static inline uint32_t ApplyUint32Div(uint32_t num, uint32_t mul,
|
||||
uint32_t extradata) {
|
||||
IDivExtraInfo extra;
|
||||
|
||||
extra.value_ = extradata;
|
||||
|
||||
uint32_t result = ((uint64_t)(num) * (uint64_t)mul) >> 32;
|
||||
if (extra.info.add_) {
|
||||
uint32_t addend = result + num;
|
||||
addend = ((addend < result ? 0x80000000 : 0) | addend);
|
||||
result = addend;
|
||||
}
|
||||
return result >> extra.info.shift_;
|
||||
}
|
||||
|
||||
static inline uint32_t ApplyUint32UMod(uint32_t num, uint32_t mul,
|
||||
uint32_t extradata, uint32_t original) {
|
||||
uint32_t dived = ApplyUint32Div(num, mul, extradata);
|
||||
unsigned result = num - (dived * original);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
struct MagicDiv {
|
||||
uint32_t multiplier_;
|
||||
uint32_t extradata_;
|
||||
MagicDiv() : multiplier_(0), extradata_(0) {}
|
||||
MagicDiv(uint32_t original) {
|
||||
multiplier_ = PregenerateUint32Div(original, extradata_);
|
||||
}
|
||||
|
||||
uint32_t Apply(uint32_t numerator) const {
|
||||
return ApplyUint32Div(numerator, multiplier_, extradata_);
|
||||
}
|
||||
};
|
||||
} // namespace divisors
|
||||
} // namespace xe
|
||||
|
||||
#endif // XENIA_BASE_MATH_H_
|
||||
|
||||
@@ -672,25 +672,58 @@ static void Prefetch<PrefetchTag::Level1>(const void* addr) {
|
||||
#define XE_MSVC_REORDER_BARRIER() static_cast<void>(0)
|
||||
#endif
|
||||
#if XE_ARCH_AMD64 == 1
|
||||
|
||||
union alignas(XE_HOST_CACHE_LINE_SIZE) CacheLine {
|
||||
struct {
|
||||
__m256 low32;
|
||||
__m256 high32;
|
||||
};
|
||||
struct {
|
||||
__m128i xmms[4];
|
||||
};
|
||||
float floats[XE_HOST_CACHE_LINE_SIZE / sizeof(float)];
|
||||
};
|
||||
XE_FORCEINLINE
|
||||
static void WriteLineNT(void* destination, const void* source) {
|
||||
assert((reinterpret_cast<uintptr_t>(destination) & 63ULL) == 0);
|
||||
__m256i low = _mm256_loadu_si256((const __m256i*)source);
|
||||
__m256i high = _mm256_loadu_si256(&((const __m256i*)source)[1]);
|
||||
XE_MSVC_REORDER_BARRIER();
|
||||
_mm256_stream_si256((__m256i*)destination, low);
|
||||
_mm256_stream_si256(&((__m256i*)destination)[1], high);
|
||||
static void WriteLineNT(CacheLine* XE_RESTRICT destination,
|
||||
const CacheLine* XE_RESTRICT source) {
|
||||
assert_true((reinterpret_cast<uintptr_t>(destination) & 63ULL) == 0);
|
||||
__m256 low = _mm256_loadu_ps(&source->floats[0]);
|
||||
__m256 high = _mm256_loadu_ps(&source->floats[8]);
|
||||
_mm256_stream_ps(&destination->floats[0], low);
|
||||
_mm256_stream_ps(&destination->floats[8], high);
|
||||
}
|
||||
|
||||
XE_FORCEINLINE
|
||||
static void ReadLineNT(void* destination, const void* source) {
|
||||
assert((reinterpret_cast<uintptr_t>(source) & 63ULL) == 0);
|
||||
__m256i low = _mm256_stream_load_si256((const __m256i*)source);
|
||||
__m256i high = _mm256_stream_load_si256(&((const __m256i*)source)[1]);
|
||||
XE_MSVC_REORDER_BARRIER();
|
||||
_mm256_storeu_si256((__m256i*)destination, low);
|
||||
_mm256_storeu_si256(&((__m256i*)destination)[1], high);
|
||||
static void ReadLineNT(CacheLine* XE_RESTRICT destination,
|
||||
const CacheLine* XE_RESTRICT source) {
|
||||
assert_true((reinterpret_cast<uintptr_t>(source) & 63ULL) == 0);
|
||||
|
||||
__m128i first = _mm_stream_load_si128(&source->xmms[0]);
|
||||
__m128i second = _mm_stream_load_si128(&source->xmms[1]);
|
||||
__m128i third = _mm_stream_load_si128(&source->xmms[2]);
|
||||
__m128i fourth = _mm_stream_load_si128(&source->xmms[3]);
|
||||
|
||||
destination->xmms[0] = first;
|
||||
destination->xmms[1] = second;
|
||||
destination->xmms[2] = third;
|
||||
destination->xmms[3] = fourth;
|
||||
}
|
||||
XE_FORCEINLINE
|
||||
static void ReadLine(CacheLine* XE_RESTRICT destination,
|
||||
const CacheLine* XE_RESTRICT source) {
|
||||
assert_true((reinterpret_cast<uintptr_t>(source) & 63ULL) == 0);
|
||||
__m256 low = _mm256_loadu_ps(&source->floats[0]);
|
||||
__m256 high = _mm256_loadu_ps(&source->floats[8]);
|
||||
_mm256_storeu_ps(&destination->floats[0], low);
|
||||
_mm256_storeu_ps(&destination->floats[8], high);
|
||||
}
|
||||
XE_FORCEINLINE
|
||||
static void WriteLine(CacheLine* XE_RESTRICT destination,
|
||||
const CacheLine* XE_RESTRICT source) {
|
||||
assert_true((reinterpret_cast<uintptr_t>(destination) & 63ULL) == 0);
|
||||
__m256 low = _mm256_loadu_ps(&source->floats[0]);
|
||||
__m256 high = _mm256_loadu_ps(&source->floats[8]);
|
||||
_mm256_storeu_ps(&destination->floats[0], low);
|
||||
_mm256_storeu_ps(&destination->floats[8], high);
|
||||
}
|
||||
|
||||
XE_FORCEINLINE
|
||||
@@ -699,19 +732,29 @@ XE_FORCEINLINE
|
||||
static void ReadFence() { _mm_lfence(); }
|
||||
XE_FORCEINLINE
|
||||
static void ReadWriteFence() { _mm_mfence(); }
|
||||
|
||||
#else
|
||||
|
||||
union alignas(XE_HOST_CACHE_LINE_SIZE) CacheLine {
|
||||
uint8_t bvals[XE_HOST_CACHE_LINE_SIZE];
|
||||
};
|
||||
XE_FORCEINLINE
|
||||
static void WriteLineNT(void* destination, const void* source) {
|
||||
assert((reinterpret_cast<uintptr_t>(destination) & 63ULL) == 0);
|
||||
memcpy(destination, source, 64);
|
||||
static void WriteLineNT(CacheLine* destination, const CacheLine* source) {
|
||||
memcpy(destination, source, XE_HOST_CACHE_LINE_SIZE);
|
||||
}
|
||||
|
||||
XE_FORCEINLINE
|
||||
static void ReadLineNT(void* destination, const void* source) {
|
||||
assert((reinterpret_cast<uintptr_t>(source) & 63ULL) == 0);
|
||||
memcpy(destination, source, 64);
|
||||
static void ReadLineNT(CacheLine* destination, const CacheLine* source) {
|
||||
memcpy(destination, source, XE_HOST_CACHE_LINE_SIZE);
|
||||
}
|
||||
XE_FORCEINLINE
|
||||
static void WriteLine(CacheLine* destination, const CacheLine* source) {
|
||||
memcpy(destination, source, XE_HOST_CACHE_LINE_SIZE);
|
||||
}
|
||||
XE_FORCEINLINE
|
||||
static void ReadLine(CacheLine* destination, const CacheLine* source) {
|
||||
memcpy(destination, source, XE_HOST_CACHE_LINE_SIZE);
|
||||
}
|
||||
|
||||
XE_FORCEINLINE
|
||||
static void WriteFence() {}
|
||||
XE_FORCEINLINE
|
||||
@@ -720,6 +763,47 @@ XE_FORCEINLINE
|
||||
static void ReadWriteFence() {}
|
||||
#endif
|
||||
} // namespace swcache
|
||||
|
||||
template <unsigned Size>
|
||||
static void smallcpy_const(void* destination, const void* source) {
|
||||
#if XE_ARCH_AMD64 == 1 && XE_COMPILER_MSVC == 1
|
||||
if constexpr ((Size & 7) == 0) {
|
||||
__movsq((unsigned long long*)destination, (const unsigned long long*)source,
|
||||
Size / 8);
|
||||
} else if constexpr ((Size & 3) == 0) {
|
||||
__movsd((unsigned long*)destination, (const unsigned long*)source,
|
||||
Size / 4);
|
||||
// dont even bother with movsw, i think the operand size override prefix
|
||||
// slows it down
|
||||
} else {
|
||||
__movsb((unsigned char*)destination, (const unsigned char*)source, Size);
|
||||
}
|
||||
#else
|
||||
memcpy(destination, source, Size);
|
||||
#endif
|
||||
}
|
||||
template <unsigned Size>
|
||||
static void smallset_const(void* destination, unsigned char fill_value) {
|
||||
#if XE_ARCH_AMD64 == 1 && XE_COMPILER_MSVC == 1
|
||||
if constexpr ((Size & 7) == 0) {
|
||||
unsigned long long fill =
|
||||
static_cast<unsigned long long>(fill_value) * 0x0101010101010101ULL;
|
||||
|
||||
__stosq((unsigned long long*)destination, fill, Size / 8);
|
||||
} else if constexpr ((Size & 3) == 0) {
|
||||
static constexpr unsigned long fill =
|
||||
static_cast<unsigned long>(fill_value) * 0x01010101U;
|
||||
__stosd((unsigned long*)destination, fill, Size / 4);
|
||||
// dont even bother with movsw, i think the operand size override prefix
|
||||
// slows it down
|
||||
} else {
|
||||
__stosb((unsigned char*)destination, fill_value, Size);
|
||||
}
|
||||
#else
|
||||
memset(destination, fill_value, Size);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace xe
|
||||
|
||||
#endif // XENIA_BASE_MEMORY_H_
|
||||
|
||||
@@ -59,16 +59,8 @@ class RingBuffer {
|
||||
// subtract instead
|
||||
void set_read_offset(size_t offset) { read_offset_ = offset % capacity_; }
|
||||
ring_size_t read_count() const {
|
||||
// chrispy: these branches are unpredictable
|
||||
#if 0
|
||||
if (read_offset_ == write_offset_) {
|
||||
return 0;
|
||||
} else if (read_offset_ < write_offset_) {
|
||||
return write_offset_ - read_offset_;
|
||||
} else {
|
||||
return (capacity_ - read_offset_) + write_offset_;
|
||||
}
|
||||
#else
|
||||
// chrispy: these branches are unpredictable
|
||||
|
||||
ring_size_t read_offs = read_offset_;
|
||||
ring_size_t write_offs = write_offset_;
|
||||
ring_size_t cap = capacity_;
|
||||
@@ -77,14 +69,6 @@ class RingBuffer {
|
||||
ring_size_t wrap_read_count = (cap - read_offs) + write_offs;
|
||||
|
||||
ring_size_t comparison_value = read_offs <= write_offs;
|
||||
#if 0
|
||||
size_t selector =
|
||||
static_cast<size_t>(-static_cast<ptrdiff_t>(comparison_value));
|
||||
offset_delta &= selector;
|
||||
|
||||
wrap_read_count &= ~selector;
|
||||
return offset_delta | wrap_read_count;
|
||||
#else
|
||||
|
||||
if (XE_LIKELY(read_offs <= write_offs)) {
|
||||
return offset_delta; // will be 0 if they are equal, semantically
|
||||
@@ -93,8 +77,6 @@ class RingBuffer {
|
||||
} else {
|
||||
return wrap_read_count;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
ring_size_t write_offset() const { return write_offset_; }
|
||||
@@ -116,9 +98,9 @@ class RingBuffer {
|
||||
void AdvanceWrite(size_t count);
|
||||
|
||||
struct ReadRange {
|
||||
const uint8_t* first;
|
||||
const uint8_t* XE_RESTRICT first;
|
||||
|
||||
const uint8_t* second;
|
||||
const uint8_t* XE_RESTRICT second;
|
||||
ring_size_t first_length;
|
||||
ring_size_t second_length;
|
||||
};
|
||||
@@ -126,9 +108,11 @@ class RingBuffer {
|
||||
void EndRead(ReadRange read_range);
|
||||
|
||||
/*
|
||||
BeginRead, but if there is a second Range it will prefetch all lines of it
|
||||
BeginRead, but if there is a second Range it will prefetch all lines of
|
||||
it
|
||||
|
||||
this does not prefetch the first range, because software prefetching can do that faster than we can
|
||||
this does not prefetch the first range, because software
|
||||
prefetching can do that faster than we can
|
||||
*/
|
||||
template <swcache::PrefetchTag tag>
|
||||
XE_FORCEINLINE ReadRange BeginPrefetchedRead(size_t count) {
|
||||
@@ -138,7 +122,7 @@ class RingBuffer {
|
||||
ring_size_t numlines =
|
||||
xe::align<ring_size_t>(range.second_length, XE_HOST_CACHE_LINE_SIZE) /
|
||||
XE_HOST_CACHE_LINE_SIZE;
|
||||
//chrispy: maybe unroll?
|
||||
// chrispy: maybe unroll?
|
||||
for (ring_size_t i = 0; i < numlines; ++i) {
|
||||
swcache::Prefetch<tag>(range.second + (i * XE_HOST_CACHE_LINE_SIZE));
|
||||
}
|
||||
@@ -187,7 +171,7 @@ class RingBuffer {
|
||||
}
|
||||
|
||||
private:
|
||||
uint8_t* buffer_ = nullptr;
|
||||
uint8_t* XE_RESTRICT buffer_ = nullptr;
|
||||
ring_size_t capacity_ = 0;
|
||||
ring_size_t read_offset_ = 0;
|
||||
ring_size_t write_offset_ = 0;
|
||||
|
||||
87
src/xenia/base/split_map.h
Normal file
87
src/xenia/base/split_map.h
Normal file
@@ -0,0 +1,87 @@
|
||||
/**
|
||||
******************************************************************************
|
||||
* Xenia : Xbox 360 Emulator Research Project *
|
||||
******************************************************************************
|
||||
* Copyright 2019 Ben Vanik. All rights reserved. *
|
||||
* Released under the BSD license - see LICENSE in the root for more details. *
|
||||
******************************************************************************
|
||||
*/
|
||||
|
||||
#ifndef XENIA_BASE_SPLIT_MAP_H_
|
||||
#define XENIA_BASE_SPLIT_MAP_H_
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
namespace xe {
|
||||
/*
|
||||
a map structure that is optimized for infrequent
|
||||
reallocation/resizing/erasure and frequent searches by key implemented as 2
|
||||
std::vectors, one of the keys and one of the values
|
||||
*/
|
||||
template <typename TKey, typename TValue>
|
||||
class split_map {
|
||||
using key_vector = std::vector<TKey>;
|
||||
using value_vector = std::vector<TValue>;
|
||||
|
||||
key_vector keys_;
|
||||
value_vector values_;
|
||||
|
||||
public:
|
||||
using my_type = split_map<TKey, TValue>;
|
||||
|
||||
uint32_t IndexForKey(const TKey& k) {
|
||||
auto lbound = std::lower_bound(keys_.begin(), keys_.end(), k);
|
||||
return static_cast<uint32_t>(lbound - keys_.begin());
|
||||
}
|
||||
|
||||
uint32_t size() const { return static_cast<uint32_t>(keys_.size()); }
|
||||
key_vector& Keys() { return keys_; }
|
||||
value_vector& Values() { return values_; }
|
||||
void clear() {
|
||||
keys_.clear();
|
||||
values_.clear();
|
||||
}
|
||||
void resize(uint32_t new_size) {
|
||||
keys_.resize(static_cast<size_t>(new_size));
|
||||
values_.resize(static_cast<size_t>(new_size));
|
||||
}
|
||||
|
||||
void reserve(uint32_t new_size) {
|
||||
keys_.reserve(static_cast<size_t>(new_size));
|
||||
values_.reserve(static_cast<size_t>(new_size));
|
||||
}
|
||||
const TKey* KeyAt(uint32_t index) const {
|
||||
if (index == size()) {
|
||||
return nullptr;
|
||||
} else {
|
||||
return &keys_[index];
|
||||
}
|
||||
}
|
||||
const TValue* ValueAt(uint32_t index) const {
|
||||
if (index == size()) {
|
||||
return nullptr;
|
||||
} else {
|
||||
return &values_[index];
|
||||
}
|
||||
}
|
||||
|
||||
void InsertAt(TKey k, TValue v, uint32_t idx) {
|
||||
uint32_t old_size = size();
|
||||
|
||||
bool needs_shiftup = idx != old_size;
|
||||
|
||||
values_.insert(values_.begin() + idx, v);
|
||||
keys_.insert(keys_.begin() + idx, k);
|
||||
}
|
||||
void EraseAt(uint32_t idx) {
|
||||
uint32_t old_size = size();
|
||||
if (idx == old_size) {
|
||||
return; // trying to erase nonexistent entry
|
||||
} else {
|
||||
values_.erase(values_.begin() + idx);
|
||||
keys_.erase(keys_.begin() + idx);
|
||||
}
|
||||
}
|
||||
};
|
||||
} // namespace xe
|
||||
|
||||
#endif // XENIA_BASE_SPLIT_MAP_H_
|
||||
Reference in New Issue
Block a user