Skip to content

Commit

Permalink
unified trace into device layer (Xilinx#2968)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hackwa
hackwa authored Mar 10, 2020
1 parent 6c8b512 commit b195f42
Show file tree
Hide file tree
Showing 4 changed files with 133 additions and 257 deletions.
96 changes: 58 additions & 38 deletions ...profile/plugin/ocl/ocl_device_offload.cpp → ...p/profile/device/device_trace_offload.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,50 +16,46 @@

#define XDP_SOURCE

#include "ocl_device_offload.h"
#include "device_trace_offload.h"
#include "xdp/profile/device/tracedefs.h"

namespace xdp {

OclDeviceOffload::OclDeviceOffload(xdp::DeviceIntf* dInt,
DeviceTraceOffload::DeviceTraceOffload(xdp::DeviceIntf* dInt,
std::shared_ptr<RTProfile> ProfileMgr,
const std::string& device_name,
const std::string& binary_name,
uint64_t sleep_interval_ms,
uint64_t trbuf_sz,
bool start_thread)
: status(DeviceOffloadStatus::IDLE),
sleep_interval_ms(sleep_interval_ms),
: sleep_interval_ms(sleep_interval_ms),
m_trbuf_alloc_sz(trbuf_sz),
dev_intf(dInt),
prof_mgr(ProfileMgr),
device_name(device_name),
binary_name(binary_name)

{
// Select appropriate reader
if(dev_intf->hasFIFO()) {
m_read_trace = std::bind(&OclDeviceOffload::read_trace_fifo, this);
if(has_fifo()) {
m_read_trace = std::bind(&DeviceTraceOffload::read_trace_fifo, this);
} else {
m_read_trace = std::bind(&OclDeviceOffload::read_trace_s2mm, this);
m_read_trace = std::bind(&DeviceTraceOffload::read_trace_s2mm, this);
}

if (start_thread) {
start_offload();
}

m_trace_vector = { } ;
}

OclDeviceOffload::~OclDeviceOffload()
DeviceTraceOffload::~DeviceTraceOffload()
{
stop_offload();
if (offload_thread.joinable()) {
offload_thread.join();
}
}

void OclDeviceOffload::offload_device_continuous()
void DeviceTraceOffload::offload_device_continuous()
{
if (!read_trace_init())
return;
Expand All @@ -75,89 +71,114 @@ void OclDeviceOffload::offload_device_continuous()
read_trace_end();
}

bool OclDeviceOffload::should_continue()
bool DeviceTraceOffload::should_continue()
{
std::lock_guard<std::mutex> lock(status_lock);
return status == DeviceOffloadStatus::RUNNING;
return status == OffloadThreadStatus::RUNNING;
}

void OclDeviceOffload::start_offload()
void DeviceTraceOffload::start_offload()
{
std::lock_guard<std::mutex> lock(status_lock);
status = DeviceOffloadStatus::RUNNING;
offload_thread = std::thread(&OclDeviceOffload::offload_device_continuous, this);
status = OffloadThreadStatus::RUNNING;
offload_thread = std::thread(&DeviceTraceOffload::offload_device_continuous, this);
}

void OclDeviceOffload::stop_offload()
void DeviceTraceOffload::stop_offload()
{
std::lock_guard<std::mutex> lock(status_lock);
status = DeviceOffloadStatus::STOPPING;
status = OffloadThreadStatus::STOPPING;
}

void OclDeviceOffload::train_clock()
void DeviceTraceOffload::train_clock()
{
static bool force = true;
dev_intf->clockTraining(force);
// Don't force continuous training for old IP
force = false;
}

void OclDeviceOffload::read_trace_fifo()
void DeviceTraceOffload::read_trace_fifo()
{
debug_stream
<< "OclDeviceOffload::read_trace_fifo " << std::endl;
<< "DeviceTraceOffload::read_trace_fifo " << std::endl;

uint32_t num_packets = 0;

do {
m_trace_vector = {};
dev_intf->readTrace(m_trace_vector);
prof_mgr->logDeviceTrace(device_name, binary_name, m_type, m_trace_vector, false);
m_trace_vector = {};
num_packets += m_trace_vector.mLength;
} while (m_trace_vector.mLength != 0);

// Check if fifo is full
if (!m_trbuf_full) {
auto property = dev_intf->getMonitorProperties(XCL_PERF_MON_FIFO, 0);
auto fifo_size = RTUtil::getDevTraceBufferSize(property);

if (num_packets >= fifo_size)
m_trbuf_full = true;

}
}

bool OclDeviceOffload::read_trace_init()
bool DeviceTraceOffload::read_trace_init()
{
if (dev_intf->hasTs2mm()) {
if (has_ts2mm()) {
return init_s2mm();
}
// reset flags
m_trbuf_full = false;
return true;
}

void OclDeviceOffload::read_trace_end()
void DeviceTraceOffload::read_trace_end()
{
// Trace logger will clear it's state and add approximations for pending
// events
// events
m_trace_vector = {};
prof_mgr->logDeviceTrace(device_name, binary_name, m_type, m_trace_vector, true);
if (dev_intf->hasTs2mm()) {
reset_s2mm();
}
}

void OclDeviceOffload::read_trace_s2mm()
void DeviceTraceOffload::read_trace_s2mm()
{
debug_stream
<< "OclDeviceOffload::read_trace_s2mm " << std::endl;
<< "DeviceTraceOffload::read_trace_s2mm " << std::endl;

// No circular buffer support for now
if (m_trbuf_full)
return;

config_s2mm_reader(dev_intf->getWordCountTs2mm());
while (1) {
auto bytes = read_trace_s2mm_partial();
prof_mgr->logDeviceTrace(device_name, binary_name, m_type, m_trace_vector, false);
m_trace_vector = {};

if (m_trbuf_sz == m_trbuf_alloc_sz)
m_trbuf_full = true;

if (bytes != m_trbuf_chunk_sz)
break;
}
}

uint64_t OclDeviceOffload::read_trace_s2mm_partial()
uint64_t DeviceTraceOffload::read_trace_s2mm_partial()
{
if (m_trbuf_offset >= m_trbuf_sz)
return 0;

uint64_t nBytes = m_trbuf_chunk_sz;

if ((m_trbuf_offset + m_trbuf_chunk_sz) > m_trbuf_sz)
nBytes = m_trbuf_sz - m_trbuf_offset;

debug_stream
<< "OclDeviceOffload::read_trace_s2mm_partial "
<< "DeviceTraceOffload::read_trace_s2mm_partial "
<<"Reading " << nBytes << " bytes " << std::endl;

auto start = std::chrono::steady_clock::now();
Expand All @@ -176,7 +197,7 @@ uint64_t OclDeviceOffload::read_trace_s2mm_partial()
return 0;
}

void OclDeviceOffload::config_s2mm_reader(uint64_t wordCount)
void DeviceTraceOffload::config_s2mm_reader(uint64_t wordCount)
{
// Start from previous offset
m_trbuf_offset = m_trbuf_sz;
Expand All @@ -185,16 +206,16 @@ void OclDeviceOffload::config_s2mm_reader(uint64_t wordCount)
m_trbuf_chunk_sz = MAX_TRACE_NUMBER_SAMPLES * TRACE_PACKET_SIZE;

debug_stream
<< "OclDeviceOffload::config_s2mm_reader "
<< "DeviceTraceOffload::config_s2mm_reader "
<< "Reading from 0x"
<< std::hex << m_trbuf_offset << " to 0x" << m_trbuf_sz
<< std::dec << std::endl;
}

bool OclDeviceOffload::init_s2mm()
bool DeviceTraceOffload::init_s2mm()
{
debug_stream
<< "OclDeviceOffload::init_s2mm with size : " << m_trbuf_alloc_sz
<< "DeviceTraceOffload::init_s2mm with size : " << m_trbuf_alloc_sz
<< std::endl;
/* If buffer is already allocated and still attempting to initialize again,
* then reset the TS2MM IP and free the old buffer
Expand All @@ -208,7 +229,6 @@ bool OclDeviceOffload::init_s2mm()

m_trbuf = dev_intf->allocTraceBuf(m_trbuf_alloc_sz, dev_intf->getTS2MmMemIndex());
if (!m_trbuf) {
xrt::message::send(xrt::message::severity_level::XRT_WARNING, TS2MM_WARN_MSG_ALLOC_FAIL);
return false;
}

Expand All @@ -218,9 +238,9 @@ bool OclDeviceOffload::init_s2mm()
return true;
}

void OclDeviceOffload::reset_s2mm()
void DeviceTraceOffload::reset_s2mm()
{
debug_stream << "OclDeviceOffload::reset_s2mm" << std::endl;
debug_stream << "DeviceTraceOffload::reset_s2mm" << std::endl;
if (!m_trbuf)
return;
dev_intf->resetTS2MM();
Expand Down
50 changes: 38 additions & 12 deletions ...p/profile/plugin/ocl/ocl_device_offload.h → ...xdp/profile/device/device_trace_offload.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
* under the License.
*/

#ifndef XDP_PROFILE_OFFLOAD_THREAD_H_
#define XDP_PROFILE_OFFLOAD_THREAD_H_
#ifndef XDP_PROFILE_DEVICE_TRACE_OFFLOAD_H_
#define XDP_PROFILE_DEVICE_TRACE_OFFLOAD_H_

#include <fstream>
#include <mutex>
Expand All @@ -27,10 +27,11 @@
#include "xdp/profile/plugin/ocl/xocl_plugin.h"
#include "xdp/profile/core/rt_profile.h"
#include "xclperf.h"
#include "xdp/config.h"

namespace xdp {

enum class DeviceOffloadStatus {
enum class OffloadThreadStatus {
IDLE,
RUNNING,
STOPPING,
Expand All @@ -40,26 +41,53 @@ enum class DeviceOffloadStatus {
#define debug_stream \
if(!m_debug); else std::cout

class OclDeviceOffload {
class DeviceTraceOffload {
public:
OclDeviceOffload(xdp::DeviceIntf* dInt, std::shared_ptr<RTProfile> ProfileMgr,
XDP_EXPORT
DeviceTraceOffload(xdp::DeviceIntf* dInt, std::shared_ptr<RTProfile> ProfileMgr,
const std::string& device_name, const std::string& binary_name,
uint64_t offload_sleep_ms, uint64_t trbuf_sz,
bool start_thread = true);
~OclDeviceOffload();
XDP_EXPORT
~DeviceTraceOffload();
XDP_EXPORT
void offload_device_continuous();
XDP_EXPORT
bool should_continue();
XDP_EXPORT
void start_offload();
XDP_EXPORT
void stop_offload();

public:
XDP_EXPORT
bool read_trace_init();
XDP_EXPORT
void read_trace_end();

public:
void set_trbuf_alloc_sz(uint64_t sz) {
m_trbuf_alloc_sz = sz;
};
bool trace_buffer_full() {
return m_trbuf_full;
};
bool has_fifo() {
return dev_intf->hasFIFO();
};
bool has_ts2mm() {
return dev_intf->hasTs2mm();
};
const std::string& get_device_name() {
return device_name;
}
void read_trace() {
m_read_trace();
};

private:
std::mutex status_lock;
DeviceOffloadStatus status;
OffloadThreadStatus status = OffloadThreadStatus::IDLE;
std::thread offload_thread;

uint64_t sleep_interval_ms;
Expand All @@ -70,25 +98,23 @@ class OclDeviceOffload {
std::string binary_name;
xclPerfMonType m_type = XCL_PERF_MON_MEMORY;

xclTraceResultsVector m_trace_vector;
xclTraceResultsVector m_trace_vector = {};
std::function<void()> m_read_trace;
size_t m_trbuf = 0;
uint64_t m_trbuf_sz = 0;
uint64_t m_trbuf_offset = 0;
uint64_t m_trbuf_chunk_sz = 0;

std::function<void()> m_read_trace;
void train_clock();
void read_trace_fifo();
void read_trace_end();
bool read_trace_init();

void read_trace_s2mm();
void* sync_trace_buf(uint64_t offset, uint64_t bytes);
uint64_t read_trace_s2mm_partial();
void config_s2mm_reader(uint64_t wordCount);
bool init_s2mm();
void reset_s2mm();

bool m_trbuf_full = false;
bool m_debug = false; /* Enable Output stream for log */
};

Expand Down
Loading

0 comments on commit b195f42

Please sign in to comment.