mirror/nasm
2
0
Fork 0
mirror of https://github.com/netwide-assembler/nasm.git synced 2025-04-12 18:40:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

file: improve the file access interface

Browse Source 
A number of fairly common operations are invoked way too many times,
especially when using incbin.  Drastically reduce the number of system
calls that need to be executed, and use memory mapping to reduce
unnecessary double buffering.

We could improve this further by leaving files open once used;
however, that might run into file count problems on some systems.

Still unclear is why we seem to invoke nasm_file_size() twice per pass
for incbin.

Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>
This commit is contained in:
H. Peter Anvin 2016-09-21 14:03:18 -07:00
parent c170089e7c
commit d81a235f33
7 changed files with 312 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
asm/assemble.c
View File

@ -614,21 +614,23 @@ int64_t assemble(int32_t segment, int64_t start, int bits, iflag_t cp,
size_t t = instruction->times;
off_t base = 0;
off_t len;
const void *map = NULL;
fp = nasm_open_read(fname, NF_BINARY);
fp = nasm_open_read(fname, NF_BINARY|NF_FORMAP);
if (!fp) {
nasm_error(ERR_NONFATAL, "`incbin': unable to open file `%s'",
fname);
goto done;
}
if (fseeko(fp, 0, SEEK_END) < 0) {
nasm_error(ERR_NONFATAL, "`incbin': unable to seek on file `%s'",
len = nasm_file_size(fp);
if (len == (off_t)-1) {
nasm_error(ERR_NONFATAL, "`incbin': unable to get length of file `%s'",
fname);
goto close_done;
}
len = ftello(fp);
if (instruction->eops->next) {
base = instruction->eops->next->offset;
if (base >= len) {
@ -640,37 +642,48 @@ int64_t assemble(int32_t segment, int64_t start, int bits, iflag_t cp,
len = (off_t)instruction->eops->next->next->offset;
}
}
lfmt->set_offset(data.offset);
lfmt->uplevel(LIST_INCBIN);
while (t--) {
off_t l;
if (!len)
goto end_incbin;
/* Try to map file data */
map = nasm_map_file(fp, base, len);
while (t--) {
data.insoffs = 0;
data.inslen = len;
if (fseeko(fp, base, SEEK_SET) < 0 || ferror(fp)) {
nasm_error(ERR_NONFATAL,
"`incbin': unable to seek on file `%s'",
fname);
goto end_incbin;
}
l = len;
while (l > 0) {
size_t m = l > (off_t)sizeof(buf) ? (size_t)l : sizeof(buf);
m = fread(buf, 1, m, fp);
if (!m || feof(fp)) {
/*
* This shouldn't happen unless the file
* actually changes while we are reading
* it.
*/
if (map) {
out_rawdata(&data, map, len);
} else {
off_t l = len;
if (fseeko(fp, base, SEEK_SET) < 0 || ferror(fp)) {
nasm_error(ERR_NONFATAL,
"`incbin': unexpected EOF while"
" reading file `%s'", fname);
"`incbin': unable to seek on file `%s'",
fname);
goto end_incbin;
}
out_rawdata(&data, buf, m);
l -= m;
while (l > 0) {
size_t m = l < (off_t)sizeof(buf) ? (size_t)l : sizeof(buf);
m = fread(buf, 1, m, fp);
if (!m || feof(fp)) {
/*
* This shouldn't happen unless the file
* actually changes while we are reading
* it.
*/
nasm_error(ERR_NONFATAL,
"`incbin': unexpected EOF while"
" reading file `%s'", fname);
goto end_incbin;
}
out_rawdata(&data, buf, m);
l -= m;
}
}
}
end_incbin:
@ -686,6 +699,8 @@ int64_t assemble(int32_t segment, int64_t start, int bits, iflag_t cp,
" reading file `%s'", fname);
}
close_done:
if (map)
nasm_unmap_file(map, len);
fclose(fp);
done:
;
@ -805,35 +820,28 @@ int64_t insn_size(int32_t segment, int64_t offset, int bits, iflag_t cp,
if (instruction->opcode == I_INCBIN) {
const char *fname = instruction->eops->stringval;
FILE *fp;
int64_t val = 0;
off_t len;
fp = nasm_open_read(fname, NF_BINARY);
if (!fp)
nasm_error(ERR_NONFATAL, "`incbin': unable to open file `%s'",
fname);
else if (fseek(fp, 0L, SEEK_END) < 0)
nasm_error(ERR_NONFATAL, "`incbin': unable to seek on file `%s'",
fname);
else {
len = ftell(fp);
if (instruction->eops->next) {
if (len <= (off_t)instruction->eops->next->offset) {
len = 0;
} else {
len -= instruction->eops->next->offset;
if (instruction->eops->next->next &&
len > (off_t)instruction->eops->next->next->offset) {
len = (off_t)instruction->eops->next->next->offset;
}
len = nasm_file_size_by_path(fname);
if (len == (off_t)-1) {
nasm_error(ERR_NONFATAL, "`incbin': unable to get length of file `%s'",
fname);
return 0;
}
if (instruction->eops->next) {
if (len <= (off_t)instruction->eops->next->offset) {
len = 0;
} else {
len -= instruction->eops->next->offset;
if (instruction->eops->next->next &&
len > (off_t)instruction->eops->next->next->offset) {
len = (off_t)instruction->eops->next->next->offset;
}
}
val = len;
}
if (fp)
fclose(fp);
return val;
return len;
}
/* Check to see if we need an address-size prefix */

62
asm/preproc.c
View File

@ -1505,9 +1505,19 @@ static bool in_list(const StrList *list, const char *str)
* ERR_FATAL and bombing out completely if not. It should also try
* the include path one by one until it finds the file or reaches
* the end of the path.
*
* Note: for INC_PROBE the function returns NULL at all times;
* instead look for the
*/
enum incopen_mode {
INC_NEEDED, /* File must exist */
INC_OPTIONAL, /* Missing is OK */
INC_PROBE /* Only an existence probe */
};
static FILE *inc_fopen(const char *file, StrList **dhead, StrList ***dtail,
char **found_path, bool missing_ok, enum file_flags mode)
char **found_path, enum incopen_mode omode,
enum file_flags fmode)
{
FILE *fp;
char *prefix = "";
@ -1516,6 +1526,7 @@ static FILE *inc_fopen(const char *file, StrList **dhead, StrList ***dtail,
size_t prefix_len = 0;
StrList *sl;
size_t path_len;
bool found;
while (1) {
path_len = prefix_len + len + 1;
@ -1524,29 +1535,32 @@ static FILE *inc_fopen(const char *file, StrList **dhead, StrList ***dtail,
memcpy(sl->str, prefix, prefix_len);
memcpy(sl->str+prefix_len, file, len+1);
if (found_path != NULL) {
*found_path = nasm_malloc(path_len);
memcpy(*found_path, sl->str, path_len);
}
fp = nasm_open_read(sl->str, mode);
if (fp && dhead && !in_list(*dhead, sl->str)) {
sl->next = NULL;
**dtail = sl;
*dtail = &sl->next;
if (omode == INC_PROBE) {
fp = NULL;
found = nasm_file_exists(sl->str);
} else {
nasm_free(sl);
fp = nasm_open_read(sl->str, fmode);
found = (fp != NULL);
}
if (fp)
if (found) {
if (found_path)
*found_path = nasm_strdup(sl->str);
if (dhead && !in_list(*dhead, sl->str)) {
sl->next = NULL;
**dtail = sl;
*dtail = &sl->next;
} else {
nasm_free(sl);
}
}
if (found)
return fp;
if (found_path != NULL && *found_path != NULL) {
nasm_free(*found_path);
*found_path = NULL;
}
nasm_free(sl);
if (!ip) {
if (!missing_ok)
if (omode == INC_NEEDED)
break;
prefix = NULL;
} else {
@ -1583,7 +1597,7 @@ FILE *pp_input_fopen(const char *filename, enum file_flags mode)
StrList *xsl = NULL;
StrList **xst = &xsl;
fp = inc_fopen(filename, &xsl, &xst, NULL, true, mode);
fp = inc_fopen(filename, &xsl, &xst, NULL, INC_OPTIONAL, mode);
if (xsl)
nasm_free(xsl);
return fp;
@ -2531,7 +2545,8 @@ static int do_directive(Token * tline)
inc->next = istk;
inc->conds = NULL;
found_path = NULL;
inc->fp = inc_fopen(p, dephead, &deptail, &found_path, pass == 0, NF_TEXT);
inc->fp = inc_fopen(p, dephead, &deptail, &found_path,
pass == 0 ? INC_OPTIONAL : INC_NEEDED, NF_TEXT);
if (!inc->fp) {
/* -MG given but file not found */
nasm_free(inc);
@ -3234,7 +3249,6 @@ issue_error:
case PP_PATHSEARCH:
{
FILE *fp;
StrList *xsl = NULL;
StrList **xst = &xsl;
@ -3274,11 +3288,9 @@ issue_error:
if (t->type != TOK_INTERNAL_STRING)
nasm_unquote(p, NULL);
fp = inc_fopen(p, &xsl, &xst, NULL, true, NF_TEXT);
if (fp) {
inc_fopen(p, &xsl, &xst, NULL, INC_PROBE, NF_BINARY);
if (xsl)
p = xsl->str;
fclose(fp); /* Don't actually care about the file */
}
macro_start = nasm_malloc(sizeof(*macro_start));
macro_start->next = NULL;
macro_start->text = nasm_quote(p, strlen(p));

14
configure.ac
View File

@ -114,8 +114,11 @@ AC_CHECK_HEADERS(inttypes.h)
AC_CHECK_HEADERS(strings.h)
AC_HEADER_STDBOOL
AC_CHECK_HEADERS(io.h)
AC_CHECK_HEADERS(fcntl.h)
AC_CHECK_HEADERS(unistd.h)
AC_CHECK_HEADERS(sys/param.h)
AC_CHECK_HEADERS(sys/mman.h)
AC_CHECK_HEADERS(sys/stat.h)
dnl Checks for library functions.
AC_CHECK_FUNCS(strcasecmp stricmp)
@ -136,6 +139,14 @@ AC_CHECK_FUNCS([_fseeki64])
AC_CHECK_FUNCS([ftruncate _chsize _chsize_s])
AC_CHECK_FUNCS([fileno])
AC_CHECK_FUNCS(stat)
AC_CHECK_FUNCS(fstat)
AC_CHECK_FUNCS(mmap)
AC_CHECK_FUNCS(getpagesize)
AC_CHECK_FUNCS(sysconf)
AC_CHECK_FUNCS([access faccessat])
PA_HAVE_FUNC(__builtin_ctz, (0U))
PA_HAVE_FUNC(__builtin_ctzl, (0UL))
PA_HAVE_FUNC(__builtin_ctzll, (0ULL))
@ -156,6 +167,9 @@ AC_CHECK_DECLS(strsep)
AC_CHECK_DECLS(strlcpy)
AC_CHECK_DECLS(strnlen)
dnl Check for missing types
AC_TYPE_UINTPTR_T
if test $ac_cv_prog_make_vpathok = no; then
echo Copying generated srcs into build directory to compensate for VPATH breakage
for file in macros.c insnsa.c insnsd.c insnsn.c insnsi.h version.h version.mac; do

13
include/nasmlib.h
View File

@ -445,19 +445,22 @@ enum file_flags {
NF_BINARY = 0x00000000, /* Binary file (default) */
NF_TEXT = 0x00000001, /* Text file */
NF_NONFATAL = 0x00000000, /* Don't die on open failure (default) */
NF_FATAL = 0x00000002 /* Die on open failure */
NF_FATAL = 0x00000002, /* Die on open failure */
NF_FORMAP = 0x00000004 /* Intended to use nasm_map_file() */
};
FILE *nasm_open_read(const char *filename, enum file_flags flags);
FILE *nasm_open_write(const char *filename, enum file_flags flags);
/* Probe for existence of a file */
bool nasm_file_exists(const char *filename);
#define ZERO_BUF_SIZE 4096 /* Default value */
#if defined(BUFSIZ) && (BUFSIZ > ZERO_BUF_SIZE)
# undef ZERO_BUF_SIZE
# define ZERO_BUF_SIZE BUFSIZ
#endif
extern const uint8_t zero_buffer[ZERO_BUF_SIZE];
void fwritezero(size_t bytes, FILE *fp);
/* Missing fseeko/ftello */
#ifndef HAVE_FSEEKO
@ -473,6 +476,12 @@ void fwritezero(size_t bytes, FILE *fp);
# endif
#endif
const void *nasm_map_file(FILE *fp, off_t start, off_t len);
void nasm_unmap_file(const void *p, size_t len);
off_t nasm_file_size(FILE *f);
off_t nasm_file_size_by_path(const char *pathname);
void fwritezero(off_t bytes, FILE *fp);
static inline bool overflow_general(int64_t value, int bytes)
{
int sbit;

203
nasmlib/file.c
View File

@ -36,12 +36,21 @@
#include <errno.h>
#ifdef HAVE_FCNTL_H
# include <fcntl.h>
#endif
#ifdef HAVE_SYS_STAT_H
# include <sys/stat.h>
#endif
#ifdef HAVE_IO_H
# include <io.h>
#endif
#ifdef HAVE_UNISTD_H
# include <unistd.h>
#endif
#ifdef HAVE_SYS_MMAN_H
# include <sys/mman.h>
#endif
/* Can we adjust the file size without actually writing all the bytes? */
#ifdef HAVE_FILENO /* Useless without fileno() */
@ -116,7 +125,7 @@ void fwriteaddr(uint64_t data, int size, FILE * fp)
#endif
void fwritezero(size_t bytes, FILE *fp)
void fwritezero(off_t bytes, FILE *fp)
{
size_t blksize;
@ -124,15 +133,16 @@ void fwritezero(size_t bytes, FILE *fp)
if (bytes >= BUFSIZ && !ferror(fp) && !feof(fp)) {
off_t pos = ftello(fp);
if (pos >= 0) {
pos += bytes;
if (!fflush(fp) &&
!nasm_ftruncate(fileno(fp), pos + bytes) &&
!fseeko(fp, pos+bytes, SEEK_SET))
!nasm_ftruncate(fileno(fp), pos) &&
!fseeko(fp, pos, SEEK_SET))
return;
}
}
#endif
while (bytes) {
while (bytes > 0) {
blksize = (bytes < ZERO_BUF_SIZE) ? bytes : ZERO_BUF_SIZE;
nasm_write(zero_buffer, blksize, fp);
@ -140,22 +150,25 @@ void fwritezero(size_t bytes, FILE *fp)
}
}
#ifdef __GLIBC__
/* If we are using glibc, attempt to mmap the files for speed */
# define READ_TEXT_FILE "rtm"
# define READ_BIN_FILE "rbm"
#else
# define READ_TEXT_FILE "rt"
# define READ_BIN_FILE "rb"
#endif
#define WRITE_TEXT_FILE "wt"
#define WRITE_BIN_FILE "wb"
FILE *nasm_open_read(const char *filename, enum file_flags flags)
{
FILE *f;
bool again = true;
#ifdef __GLIBC__
/*
* Try to open this file with memory mapping for speed, unless we are
* going to do it "manually" with nasm_map_file()
*/
if (!(flags & NF_FORMAP)) {
f = fopen(filename, (flags & NF_TEXT) ? "rtm" : "rbm");
again = (!f) && (errno == EINVAL); /* Not supported, try without m */
}
#endif
if (again)
f = fopen(filename, (flags & NF_TEXT) ? "rt" : "rb");
f = fopen(filename, (flags & NF_TEXT) ? READ_TEXT_FILE : READ_BIN_FILE);
if (!f && (flags & NF_FATAL))
nasm_fatal(ERR_NOFILE, "unable to open input file: `%s': %s",
filename, strerror(errno));
@ -167,7 +180,8 @@ FILE *nasm_open_write(const char *filename, enum file_flags flags)
{
FILE *f;
f = fopen(filename, (flags & NF_TEXT) ? WRITE_TEXT_FILE : WRITE_BIN_FILE);
f = fopen(filename, (flags & NF_TEXT) ? "wt" : "wb");
if (!f && (flags & NF_FATAL))
nasm_fatal(ERR_NOFILE, "unable to open output file: `%s': %s",
filename, strerror(errno));
@ -175,4 +189,157 @@ FILE *nasm_open_write(const char *filename, enum file_flags flags)
return f;
}
off_t
/*
* Report the existence of a file
*/
bool nasm_file_exists(const char *filename)
{
#if defined(HAVE_FACCESSAT) && defined(AT_EACCESS)
return faccessat(AT_FDCWD, filename, R_OK, AT_EACCESS) == 0;
#elif defined(HAVE_ACCESS)
return access(filename, R_OK) == 0;
#else
FILE *f;
f = fopen(filename, "rb");
if (f) {
fclose(f);
return true;
} else {
return false;
}
#endif
}
/*
* Report file size. This MAY move the file pointer.
*/
off_t nasm_file_size(FILE *f)
{
#if defined(HAVE_FILENO) && defined(HAVE_FSTAT)
struct stat st;
if (fstat(fileno(f), &st))
return (off_t)-1;
return st.st_size;
#else
if (fseeko(f, 0, SEEK_END))
return (off_t)-1;
return ftello(f);
#endif
}
/*
* Report file size given pathname
*/
off_t nasm_file_size_by_path(const char *pathname)
{
#ifdef HAVE_STAT
struct stat st;
if (stat(pathname, &st))
return (off_t)-1;
return st.st_size;
#else
FILE *fp;
off_t len;
fp = nasm_open_read(pathname, NF_BINARY);
if (!fp)
return (off_t)-1;
len = nasm_file_size(fp);
fclose(fp);
return len;
#endif
}
/*
* System page size
*/
/* File scope since not all compilers like static data in inline functions */
static size_t nasm_pagemask;
static size_t get_pagemask(void)
{
size_t ps = 0;
# if defined(HAVE_SYSCONF) && defined(_SC_PAGESIZE)
ps = sysconf(_SC_PAGESIZE);
# elif defined(HAVE_GETPAGESIZE)
ps = getpagesize();
# endif
nasm_pagemask = ps = is_power2(ps) ? (ps - 1) : 0;
return ps;
}
static inline size_t pagemask(void)
{
size_t pm = nasm_pagemask;
if (unlikely(!pm))
return get_pagemask();
return pm;
}
/*
* Try to map an input file into memory
*/
const void *nasm_map_file(FILE *fp, off_t start, off_t len)
{
#if defined(HAVE_FILENO) && defined(HAVE_MMAP)
const char *p;
off_t astart; /* Aligned start */
size_t salign; /* Amount of start adjustment */
size_t alen; /* Aligned length */
const size_t page_mask = pagemask();
if (unlikely(!page_mask))
return NULL; /* Page size undefined? */
if (unlikely(!len))
return NULL; /* Mapping nothing... */
if (unlikely(len != (off_t)(size_t)len))
return NULL; /* Address space insufficient */
astart = start & ~(off_t)page_mask;
salign = start - astart;
alen = (len + salign + page_mask) & ~page_mask;
p = mmap(NULL, alen, PROT_READ, MAP_SHARED, fileno(fp), astart);
return unlikely(p == MAP_FAILED) ? NULL : p + salign;
#else
/* XXX: add Windows support? */
return NULL;
#endif
}
/*
* Unmap an input file
*/
void nasm_unmap_file(const void *p, size_t len)
{
#if defined(HAVE_FILENO) && defined(HAVE_MMAP)
const size_t page_mask = pagemask();
uintptr_t astart;
size_t salign;
size_t alen;
if (unlikely(!page_mask))
return;
astart = (uintptr_t)p & ~(uintptr_t)page_mask;
salign = (uintptr_t)p - astart;
alen = (len + salign + page_mask) & ~page_mask;
munmap((void *)astart, alen);
#endif
}

5
test/incbin.asm Normal file
View File

@ -0,0 +1,5 @@
db '*** ONCE ***', 0Ah
incbin "incbin.data",32
db '*** TWELVE ***', 0Ah
times 12 incbin "incbin.data",32

2
test/incbin.data Normal file
View File

@ -0,0 +1,2 @@
This is the wrong line of text.
This is the right line of text.