Skip to content

Commit

Permalink
Merge pull request simongog#86 from simongog/fix_hugepages
Browse files Browse the repository at this point in the history 
Fix hugepages
  • Loading branch information
@simongog
simongog committed Sep 12, 2013
2 parents a78bdf6 + 46bf04e commit 112aabf
Show file tree
Hide file tree
Showing 8 changed files with 47 additions and 180 deletions.
2 changes: 1 addition & 1 deletion CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ set(LIBRARY_VERSION "${LIBRARY_VERSION_MAJOR}.${LIBRARY_VERSION_MINOR}")
set(LIBRARY_VERSION_FULL "${LIBRARY_VERSION}.${LIBRARY_VERSION_PATCH}")

append_cxx_compiler_flags("-std=c++11 -Wall -DNDEBUG" "GCC" CMAKE_CXX_FLAGS)
append_cxx_compiler_flags("-O9 -ffast-math -funroll-loops" "GCC" CMAKE_CXX_OPT_FLAGS)
append_cxx_compiler_flags("-O0 -g -ffast-math -funroll-loops" "GCC" CMAKE_CXX_OPT_FLAGS)
append_cxx_compiler_flags("-std=c++11 -funroll-loops -DNDEBUG -stdlib=libc++" "CLANG" CMAKE_CXX_FLAGS)
append_cxx_compiler_flags("-O3 -ffast-math -funroll-loops" "CLANG" CMAKE_CXX_OPT_FLAGS)

Expand Down
4 changes: 2 additions & 2 deletions benchmark/indexing_count/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ results/%: $(BUILD_EXECS) $(QUERY_EXECS) $(PATTERNS) $(INDEXES)
@echo "# COMPILE_ID = $(COMPILE_ID)" >> $@
@echo "# test_case = $(TC_NAME)" >> $@
@echo "Run timing for $(IDX_ID).$(COMPILE_ID) on $(TC_ID)"
@$(BIN_DIR)/query_idx_$(IDX_ID).$(COMPILE_ID) \
$(BIN_DIR)/query_idx_$(IDX_ID).$(COMPILE_ID) \
indexes/$(TC_ID) C < $(PAT_DIR)/$(TC_ID).pattern 2>> $@


Expand All @@ -65,7 +65,7 @@ indexes/%: $(BUILD_EXECS)
$(eval IDX_ID:=$(call dim,2,$*))
$(eval TC:=$(call config_select,test_case.config,$(TC_ID),2))
@echo "Building index $(IDX_ID) on $(TC)"
@$(BIN_DIR)/build_idx_$(IDX_ID) $(TC) $(TMP_DIR) $@
$(BIN_DIR)/build_idx_$(IDX_ID) $(TC) $(TMP_DIR) $@

# info/[TC_ID].[IDX_ID]
info/%.json: $(INDEXES)
Expand Down
12 changes: 6 additions & 6 deletions benchmark/indexing_count/compile_options.config
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
# Column description (columns are separated by semicolon):
# (1) Identifier for compile configuration (consisting of letters)
# (2) Compile options
NOOPT;-std=c++11 -DPOPCOUNT_TL -O3 -funroll-loops -fomit-frame-pointer -ffast-math -DNDEBUG
NOSSE;-std=c++11 -O3 -funroll-loops -fomit-frame-pointer -ffast-math -DNDEBUG
SSE;-std=c++11 -msse4.2 -O3 -funroll-loops -fomit-frame-pointer -ffast-math -DNDEBUG
HP;-std=c++11 -DUSE_HP -msse4.2 -O3 -funroll-loops -fomit-frame-pointer -ffast-math -DNDEBUG
O1;-std=c++11 -O1 -DNDEBUG
O0;-std=c++11 -O0 -DNDEBUG
NOOPT;-std=c++11 -DPOPCOUNT_TL -O0 -g -funroll-loops -fomit-frame-pointer -ffast-math
NOSSE;-std=c++11 -O0 -g -funroll-loops -fomit-frame-pointer -ffast-math
SSE;-std=c++11 -msse4.2 -O0 -g -funroll-loops -fomit-frame-pointer -ffast-math
HP;-std=c++11 -DUSE_HP -msse4.2 -O0 -g -funroll-loops -fomit-frame-pointer -ffast-math
O1;-std=c++11 -O1
O0;-std=c++11 -O0
178 changes: 17 additions & 161 deletions benchmark/indexing_count/src/run_queries_sdsl.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,7 @@ using namespace std;
/* local headers */
void do_count(const CSA_TYPE&);
void do_locate(const CSA_TYPE&);
//void do_extract (void);
//void do_display(ulong length);
void pfile_info(ulong* length, ulong* numpatt);
//void output_char(uchar c, FILE * where);
double getTime(void);
void usage(char* progname);

Expand All @@ -52,12 +49,26 @@ int main(int argc, char* argv[])

filename = argv[1];
querytype = *argv[2];
string index_file = string(argv[1]) + "." + string(SUF);
bool mapped = false;
#ifdef USE_HP
uint64_t index_size = util::file_size(index_file);
// allocate hugepages, add 10 MiB + 3% overhead
uint64_t alloc_size = (uint64_t)(index_size*1.03+(1ULL<<20)*10);
try {
memory_manager::use_hugepages(alloc_size);
mapped = true;
} catch (...) {
std::cout<<"Unable to allocate "<<alloc_size<<" bytes ";
std::cout<<"of hugepage space on your system"<<std::endl;
}
#endif

CSA_TYPE csa;
fprintf(stderr, "# File = %s\n",(string(filename) + "." + string(SUF)).c_str());
fprintf(stderr, "# program = %s\n",string(SUF).c_str());
Load_time = getTime();
load_from_file(csa, (string(argv[1]) + "." + string(SUF)).c_str());
load_from_file(csa, index_file);
Load_time = getTime() - Load_time;
fprintf(stderr, "# Load_index_time_in_sec = %.2f\n", Load_time);
std::cerr<<"# text_size = " << csa.size()-1 << std::endl;
Expand All @@ -66,10 +77,7 @@ int main(int argc, char* argv[])
Text_length = csa.size()-1; // -1 since we added a sentinel character
/* Index_size /=1024; */
fprintf(stderr, "# Index_size_in_bytes = %lu\n", Index_size);
bool mapped = false;
#ifdef USE_HP
mapped = mm::map_hp();
#endif

fprintf(stderr, "# hugepages = %i\n", (int)mapped);
bool use_sse = false;
#ifdef __SSE4_2__
Expand Down Expand Up @@ -99,37 +107,10 @@ int main(int argc, char* argv[])
}
do_locate(csa);
break;
/* case EXTRACT:
if (argc > 3)
if (*argv[3] == VERBOSE) {
Verbose = 1;
fprintf(stdout,"%c", EXTRACT);
}
do_extract();
break;
case DISPLAY:
if (argc < 4) {
usage(argv[0]);
exit (1);
}
if (argc > 4)
if (*argv[4] == VERBOSE){
Verbose = 1;
fprintf(stdout,"%c", DISPLAY);
}
do_display((ulong) atol(argv[3]));
break;
*/ default:
default:
fprintf(stderr, "Unknow option: main ru\n");
exit(1);
}
#ifdef USE_HP
if (mapped) {
mm::unmap_hp();
}
#endif
return 0;
}

Expand Down Expand Up @@ -235,80 +216,6 @@ do_locate(const CSA_TYPE& csa)
free(pattern);
}


/*
void do_display(ulong numc) {
int error = 0;
ulong numocc, length, i, *snippet_len, tot_numcharext = 0, numpatt;
double time, tot_time = 0;
uchar *pattern, *snippet_text;
pfile_info (&length, &numpatt);
pattern = (uchar *) malloc (sizeof (uchar) * (length));
if (pattern == NULL)
{
fprintf (stderr, "Error: cannot allocate\n");
exit (1);
}
fprintf(stderr, "Snippet length %lu\n", numc);
while (numpatt)
{
if (fread (pattern, sizeof (*pattern), length, stdin) != length)
{
fprintf (stderr, "Error: cannot read patterns file\n");
perror ("run_queries");
exit (1);
}
// Display
time = getTime ();
error = display (Index, pattern, length, numc, &numocc,
&snippet_text, &snippet_len);
IFERROR (error);
tot_time += (getTime () - time);
if (Verbose) {
ulong j, len = length + 2*numc;
char blank = '\0';
fwrite(&length, sizeof(length), 1, stdout);
fwrite(pattern, sizeof(*pattern), length, stdout);
fwrite(&numocc, sizeof(numocc), 1, stdout);
fwrite(&len, sizeof(len), 1, stdout);
for (i = 0; i < numocc; i++){
fwrite(snippet_text+len*i,sizeof(uchar),snippet_len[i],stdout);
for(j=snippet_len[i];j<len;j++)
fwrite(&blank,sizeof(uchar),1,stdout);
}
}
numpatt--;
for(i=0; i<numocc; i++) {
tot_numcharext += snippet_len[i];
}
if (numocc) {
free (snippet_len);
free (snippet_text);
}
}
fprintf (stderr, "#Total_num_chars_extracted = %lu\n", tot_numcharext);
fprintf (stderr, "#Display_time_in_sec = %.2f secs\n", tot_time);
fprintf (stderr, "#Time_display/Tot_num_chars = %.4f\n\n", (tot_time*1000) / tot_numcharext);
fprintf (stderr, "#(Load_time+Time_display)/Tot_num_chars = %.4f\n\n", ((Load_time+tot_time)*1000) / tot_numcharext);
free (pattern);
}
*/


/* Open patterns file and read header */
void
pfile_info(ulong* length, ulong* numpatt)
Expand Down Expand Up @@ -338,57 +245,6 @@ pfile_info(ulong* length, ulong* numpatt)

}

/*
void
do_extract()
{
int error = 0;
uchar *text, orig_file[257];
ulong num_pos, from, to, numchars, readen, tot_ext = 0;
double time, tot_time = 0;
error = fscanf(stdin, "# number=%lu length=%lu file=%s\n", &num_pos, &numchars, orig_file);
if (error != 3)
{
fprintf (stderr, "Error: Intervals file header is not correct\n");
perror ("run_queries");
exit (1);
}
fprintf(stderr, "# number=%lu length=%lu file=%s\n", num_pos, numchars, orig_file);
while(num_pos) {
if(fscanf(stdin,"%lu,%lu\n", &from, &to) != 2) {
fprintf(stderr, "Cannot read correctly intervals file\n");
exit(1);
}
time = getTime();
error = extract(Index, from, to, &text, &readen);
IFERROR(error);
tot_time += (getTime() - time);
tot_ext += readen;
if (Verbose) {
fwrite(&from,sizeof(ulong),1,stdout);
fwrite(&readen,sizeof(ulong),1,stdout);
fwrite(text,sizeof(uchar),readen, stdout);
}
num_pos--;
free(text);
}
fprintf (stderr, "# Total_num_chars_extracted = %lu\n", tot_ext);
fprintf (stderr, "# Extract_time_in_sec = %.2f\n", tot_time);
fprintf (stderr, "# Extract_time/Num_chars_extracted = %.4f\n\n",
(tot_time * 1000) / tot_ext);
fprintf (stderr, "(Load_time+Extract_time)/Num_chars_extracted = %.4f\n\n",
((Load_time+tot_time) * 1000) / tot_ext);
}
*/

double
getTime(void)
{
Expand Down
2 changes: 1 addition & 1 deletion include/sdsl/int_vector.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -304,7 +304,7 @@ class int_vector
int_vector(std::initializer_list<t_T> il) : int_vector() {
resize(il.size());
size_type idx = 0;
for (auto x : il) {
for (auto x : il) {
(*this)[idx++] = x;
}
}
Expand Down
27 changes: 18 additions & 9 deletions include/sdsl/memory_management.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ class hugepage_allocator
mm_block_t* last_block();
void print_heap();
public:
void init(size_t size_in_bytes) {
void init(SDSL_UNUSED size_t size_in_bytes) {
#ifdef MAP_HUGETLB
m_total_size = size_in_bytes;
m_base = (uint8_t*) mmap(nullptr, m_total_size,
Expand All @@ -158,6 +158,16 @@ class hugepage_allocator
void* mm_realloc(void* ptr, size_t size);
void* mm_alloc(size_t size_in_bytes);
void mm_free(void* ptr);
bool in_address_space(void* ptr) {
// check if ptr is in the hugepage address space
if (ptr == nullptr) {
return true;
}
if (ptr >= m_base && ptr < m_top) {
return true;
}
return false;
}
static hugepage_allocator& the_allocator() {
static hugepage_allocator a;
return a;
Expand All @@ -184,15 +194,15 @@ class memory_manager
}
static void free_mem(uint64_t* ptr) {
auto& m = the_manager();
if (m.hugepages) {
if (m.hugepages and hugepage_allocator::the_allocator().in_address_space(ptr)) {
hugepage_allocator::the_allocator().mm_free(ptr);
} else {
std::free(ptr);
}
}
static uint64_t* realloc_mem(uint64_t* ptr,size_t size) {
auto& m = the_manager();
if (m.hugepages) {
if (m.hugepages and hugepage_allocator::the_allocator().in_address_space(ptr)) {
return (uint64_t*) hugepage_allocator::the_allocator().mm_realloc(ptr,size);
} else {
return (uint64_t*) realloc(ptr,size);
Expand All @@ -201,16 +211,16 @@ class memory_manager
public:
static void use_hugepages(size_t bytes) {
auto& m = the_manager();
m.hugepages = true;
hugepage_allocator::the_allocator().init(bytes);
m.hugepages = true;
}
template<class t_vec>
static void resize(t_vec& v, const typename t_vec::size_type size) {
int64_t old_size_in_bytes = ((v.m_size+63)>>6)<<3;
int64_t new_size_in_bytes = ((size+63)>>6)<<3;
uint64_t old_size_in_bytes = ((v.m_size+63)>>6)<<3;
uint64_t new_size_in_bytes = ((size+63)>>6)<<3;
bool do_realloc = old_size_in_bytes != new_size_in_bytes;
v.m_size = size;
if (do_realloc || new_size_in_bytes == 0) {
if (do_realloc || v.m_data == nullptr) {
// Note that we allocate 8 additional bytes if m_size % 64 == 0.
// We need this padding since rank data structures do a memory
// access to this padding to answer rank(size()) if size()%64 ==0.
Expand All @@ -230,7 +240,7 @@ class memory_manager

// update stats
if (do_realloc) {
memory_monitor::record(new_size_in_bytes-old_size_in_bytes);
memory_monitor::record((int64_t)new_size_in_bytes-(int64_t)old_size_in_bytes);
}
}
}
Expand All @@ -248,7 +258,6 @@ class memory_manager
}
};


} // end namespace

#endif
1 change: 1 addition & 0 deletions include/sdsl/rrr_vector.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ class rrr_vector

//! Move constructor
rrr_vector(rrr_vector&& rrr) : m_size(std::move(rrr.m_size)),
m_bt(std::move(rrr.m_bt)),
m_btnr(std::move(rrr.m_btnr)), m_btnrp(std::move(rrr.m_btnrp)),
m_rank(std::move(rrr.m_rank)), m_invert(std::move(rrr.m_invert)) {}

Expand Down
1 change: 1 addition & 0 deletions include/sdsl/rrr_vector_15.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ class rrr_vector<15, t_rac, t_k>

//! Move constructor
rrr_vector(rrr_vector&& rrr) : m_size(std::move(rrr.m_size)),
m_bt(std::move(rrr.m_bt)),
m_btnr(std::move(rrr.m_btnr)), m_btnrp(std::move(rrr.m_btnrp)),
m_rank(std::move(rrr.m_rank)) {}

Expand Down

0 comments on commit 112aabf

Please sign in to comment.