lsm/mdbtools
1
0
Fork 0
mirror of https://github.com/mdbtools/mdbtools.git synced 2025-04-05 20:31:00 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

New mdb_set_bind_size function overrides MDB_BIND_SIZE

Browse Source 
This should fix long-standing complaints about the default bind size
without causing undue memory inflation in existing applications.

Could make this adjustable on the command line later.

Supersedes:

https://github.com/mdbtools/mdbtools/pull/137
This commit is contained in:
Evan Miller 2020-09-02 14:30:07 -04:00
parent b7dd44d0d4
commit fb960553e6
7 changed files with 46 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
include/mdbtools.h
View File

@ -54,7 +54,7 @@
#define MDB_MAX_IDX_COLS 10
#define MDB_CATALOG_PG 18
#define MDB_MEMO_OVERHEAD 12
#define MDB_BIND_SIZE 16384
#define MDB_BIND_SIZE 16384 // override with mdb_set_bind_size(MdbHandle*, size_t)
// This attribute is not supported by all compilers:
// M$VC see http://stackoverflow.com/questions/1113409/attribute-constructor-equivalent-in-vc
@ -269,6 +269,7 @@ typedef struct {
unsigned char pg_buf[MDB_PGSIZE];
unsigned char alt_pg_buf[MDB_PGSIZE];
MdbFormatConstants *fmt;
size_t bind_size;
char date_fmt[64];
const char *boolean_false_value;
const char *boolean_true_value;
@ -512,6 +513,7 @@ int mdb_col_disp_size(MdbColumn *col);
size_t mdb_ole_read_next(MdbHandle *mdb, MdbColumn *col, void *ole_ptr);
size_t mdb_ole_read(MdbHandle *mdb, MdbColumn *col, void *ole_ptr, size_t chunk_size);
void* mdb_ole_read_full(MdbHandle *mdb, MdbColumn *col, size_t *size);
void mdb_set_bind_size(MdbHandle *mdb, size_t bind_size);
void mdb_set_date_fmt(MdbHandle *mdb, const char *);
void mdb_set_boolean_fmt_words(MdbHandle *mdb);
void mdb_set_boolean_fmt_numbers(MdbHandle *mdb);

2
src/libmdb/backend.c
View File

@ -623,7 +623,7 @@ mdb_get_relationships(MdbHandle *mdb, const gchar *dbnamespace, const char* tabl
mdb_read_columns(mdb->relationships_table);
for (i=0;i<5;i++) {
bound[i] = (char *) g_malloc0(MDB_BIND_SIZE);
bound[i] = (char *) g_malloc0(mdb->bind_size);
}
mdb_bind_column_by_name(mdb->relationships_table, "szColumn", bound[0], NULL);
mdb_bind_column_by_name(mdb->relationships_table, "szObject", bound[1], NULL);

31
src/libmdb/data.c
View File

@ -22,6 +22,7 @@
#include "mdbtools.h"
#define OFFSET_MASK 0x1fff
#define OLE_BUFFER_SIZE (MDB_BIND_SIZE*64)
char *mdb_money_to_string(MdbHandle *mdb, int start);
char *mdb_numeric_to_string(MdbHandle *mdb, int start, int prec, int scale);
@ -46,6 +47,10 @@ static const char boolean_true_number[] = "1";
static const char boolean_false_word[] = "FALSE";
static const char boolean_true_word[] = "TRUE";
void mdb_set_bind_size(MdbHandle *mdb, size_t bind_size) {
mdb->bind_size = bind_size;
}
void mdb_set_date_fmt(MdbHandle *mdb, const char *fmt)
{
snprintf(mdb->date_fmt, sizeof(mdb->date_fmt), "%s", fmt);
@ -255,7 +260,7 @@ int ret;
} else {
str = mdb_col_to_string(mdb, mdb->pg_buf, start, col->col_type, len);
}
snprintf(col->bind_ptr, MDB_BIND_SIZE, "%s", str);
snprintf(col->bind_ptr, mdb->bind_size, "%s", str);
g_free(str);
}
ret = strlen(col->bind_ptr);
@ -610,18 +615,18 @@ void*
mdb_ole_read_full(MdbHandle *mdb, MdbColumn *col, size_t *size)
{
char ole_ptr[MDB_MEMO_OVERHEAD];
char *result = malloc(MDB_BIND_SIZE * 64);
size_t result_buffer_size = MDB_BIND_SIZE * 64;
char *result = malloc(OLE_BUFFER_SIZE);
size_t result_buffer_size = OLE_BUFFER_SIZE;
size_t len, pos;
memcpy(ole_ptr, col->bind_ptr, MDB_MEMO_OVERHEAD);
len = mdb_ole_read(mdb, col, ole_ptr, MDB_BIND_SIZE * 64);
len = mdb_ole_read(mdb, col, ole_ptr, OLE_BUFFER_SIZE);
memcpy(result, col->bind_ptr, len);
pos = len;
while ((len = mdb_ole_read_next(mdb, col, ole_ptr))) {
if (pos+len >= result_buffer_size) {
result_buffer_size += MDB_BIND_SIZE * 64;
result_buffer_size += OLE_BUFFER_SIZE;
result = realloc(result, result_buffer_size);
}
memcpy(result + pos, col->bind_ptr, len);
@ -703,7 +708,7 @@ static char *mdb_memo_to_string(MdbHandle *mdb, int start, int size)
gint32 row_start, pg_row;
size_t len;
void *buf, *pg_buf = mdb->pg_buf;
char *text = (char *) g_malloc(MDB_BIND_SIZE);
char *text = (char *) g_malloc(mdb->bind_size);
if (size<MDB_MEMO_OVERHEAD) {
strcpy(text, "");
@ -723,7 +728,7 @@ static char *mdb_memo_to_string(MdbHandle *mdb, int start, int size)
if (memo_len & 0x80000000) {
/* inline memo field */
mdb_unicode2ascii(mdb, (char*)pg_buf + start + MDB_MEMO_OVERHEAD,
size - MDB_MEMO_OVERHEAD, text, MDB_BIND_SIZE);
size - MDB_MEMO_OVERHEAD, text, mdb->bind_size);
return text;
} else if (memo_len & 0x40000000) {
/* single-page memo field */
@ -740,7 +745,7 @@ static char *mdb_memo_to_string(MdbHandle *mdb, int start, int size)
pg_row & 0xff, row_start, len);
mdb_buffer_dump(buf, row_start, len);
#endif
mdb_unicode2ascii(mdb, (char*)buf + row_start, len, text, MDB_BIND_SIZE);
mdb_unicode2ascii(mdb, (char*)buf + row_start, len, text, mdb->bind_size);
return text;
} else if ((memo_len & 0xff000000) == 0) { // assume all flags in MSB
/* multi-page memo field */
@ -775,7 +780,7 @@ static char *mdb_memo_to_string(MdbHandle *mdb, int start, int size)
if (tmpoff < memo_len) {
fprintf(stderr, "Warning: incorrect memo length\n");
}
mdb_unicode2ascii(mdb, tmp, tmpoff, text, MDB_BIND_SIZE);
mdb_unicode2ascii(mdb, tmp, tmpoff, text, mdb->bind_size);
g_free(tmp);
return text;
} else {
@ -875,12 +880,12 @@ static char *
mdb_date_to_string(MdbHandle *mdb, void *buf, int start)
{
struct tm t;
char *text = (char *) g_malloc(MDB_BIND_SIZE);
char *text = (char *) g_malloc(mdb->bind_size);
double td = mdb_get_double(buf, start);
mdb_date_to_tm(td, &t);
strftime(text, MDB_BIND_SIZE, mdb->date_fmt, &t);
strftime(text, mdb->bind_size, mdb->date_fmt, &t);
return text;
}
@ -974,9 +979,9 @@ char *mdb_col_to_string(MdbHandle *mdb, void *buf, int start, int datatype, int
if (size<0) {
text = g_strdup("");
} else {
text = (char *) g_malloc(MDB_BIND_SIZE);
text = (char *) g_malloc(mdb->bind_size);
mdb_unicode2ascii(mdb, (char*)buf + start,
size, text, MDB_BIND_SIZE);
size, text, mdb->bind_size);
}
break;
case MDB_DATETIME:

1
src/libmdb/file.c
View File

@ -174,6 +174,7 @@ static MdbHandle *mdb_handle_from_stream(FILE *stream, MdbFileFlags flags) {
MdbHandle *mdb = (MdbHandle *) g_malloc0(sizeof(MdbHandle));
mdb_set_default_backend(mdb, "access");
mdb_set_date_fmt(mdb, "%x %X");
mdb_set_bind_size(mdb, MDB_BIND_SIZE);
mdb_set_boolean_fmt_numbers(mdb);
#ifdef HAVE_ICONV
mdb->iconv_in = (iconv_t)-1;

7
src/util/mdb-export.c
View File

@ -18,8 +18,7 @@
#include "mdbtools.h"
#undef MDB_BIND_SIZE
#define MDB_BIND_SIZE 200000
#define EXPORT_BIND_SIZE 200000
#define is_quote_type(x) (x==MDB_TEXT || x==MDB_OLE || x==MDB_MEMO || x==MDB_DATETIME || x==MDB_BINARY || x==MDB_REPID)
#define is_binary_type(x) (x==MDB_OLE || x==MDB_BINARY || x==MDB_REPID)
@ -191,6 +190,8 @@ main(int argc, char **argv)
if (boolean_words)
mdb_set_boolean_fmt_words(mdb);
mdb_set_bind_size(mdb, EXPORT_BIND_SIZE);
if (insert_dialect)
if (!mdb_set_default_backend(mdb, insert_dialect)) {
fputs("Invalid backend type\n", stderr);
@ -213,7 +214,7 @@ main(int argc, char **argv)
bound_lens = (int *) g_malloc(table->num_cols * sizeof(int));
for (i = 0; i < table->num_cols; i++) {
/* bind columns */
bound_values[i] = (char *) g_malloc0(MDB_BIND_SIZE);
bound_values[i] = (char *) g_malloc0(EXPORT_BIND_SIZE);
mdb_bind_column(table, i + 1, bound_values[i], &bound_lens[i]);
}
if (header_row) {

7
src/util/mdb-json.c
View File

@ -20,8 +20,7 @@
#include "base64.h"
#undef MDB_BIND_SIZE
#define MDB_BIND_SIZE 200000
#define EXPORT_BIND_SIZE 200000
#define is_quote_type(x) (x==MDB_TEXT || x==MDB_OLE || x==MDB_MEMO || x==MDB_DATETIME || x==MDB_BINARY || x==MDB_REPID)
#define is_binary_type(x) (x==MDB_OLE || x==MDB_BINARY || x==MDB_REPID)
@ -144,6 +143,8 @@ main(int argc, char **argv)
if (date_fmt)
mdb_set_date_fmt(mdb, date_fmt);
mdb_set_bind_size(mdb, EXPORT_BIND_SIZE);
table = mdb_read_table_by_name(mdb, argv[2], MDB_TABLE);
if (!table) {
fprintf(stderr, "Error: Table %s does not exist in this database.\n", argv[argc-1]);
@ -159,7 +160,7 @@ main(int argc, char **argv)
bound_lens = (int *) g_malloc(table->num_cols * sizeof(int));
for (i=0;i<table->num_cols;i++) {
/* bind columns */
bound_values[i] = (char *) g_malloc0(MDB_BIND_SIZE);
bound_values[i] = (char *) g_malloc0(EXPORT_BIND_SIZE);
mdb_bind_column(table, i+1, bound_values[i], &bound_lens[i]);
}

28
src/util/mdb-queries.c
View File

@ -33,8 +33,7 @@
#include "mdbtools.h"
#undef MDB_BIND_SIZE
#define MDB_BIND_SIZE 200000
#define QUERY_BIND_SIZE 200000
void mdb_list_queries(MdbHandle *mdb, int line_break, char *delimiter);
char * mdb_get_query_id(MdbHandle *mdb,char *query);
@ -50,21 +49,22 @@ int main (int argc, char **argv) {
int line_break=0;
int opt;
char *query_id;
size_t bind_size = QUERY_BIND_SIZE;
// variables for the msysqueries table. hopefully 256 is big enough
char *attribute = (char *) malloc(MDB_BIND_SIZE);
char *expression = (char *) malloc(MDB_BIND_SIZE);
char *flag = (char *) malloc(MDB_BIND_SIZE);
char *name1 = (char *) malloc(MDB_BIND_SIZE);
char *name2 = (char *) malloc(MDB_BIND_SIZE);
char *objectid = (char *) malloc(MDB_BIND_SIZE);
char *order = (char *) malloc(MDB_BIND_SIZE);
char *attribute = (char *) malloc(bind_size);
char *expression = (char *) malloc(bind_size);
char *flag = (char *) malloc(bind_size);
char *name1 = (char *) malloc(bind_size);
char *name2 = (char *) malloc(bind_size);
char *objectid = (char *) malloc(bind_size);
char *order = (char *) malloc(bind_size);
//variables for the generation of sql
char *sql_tables = (char *) malloc(MDB_BIND_SIZE);
char *sql_columns = (char *) malloc(MDB_BIND_SIZE);
char *sql_where = (char *) malloc(MDB_BIND_SIZE);
char *sql_sorting = (char *) malloc(MDB_BIND_SIZE);
char *sql_tables = (char *) malloc(bind_size);
char *sql_columns = (char *) malloc(bind_size);
char *sql_where = (char *) malloc(bind_size);
char *sql_sorting = (char *) malloc(bind_size);
/* see getopt(3) for more information on getopt and this will become clear */
while ((opt=getopt(argc, argv, "L1d:"))!=-1) {
@ -102,6 +102,8 @@ int main (int argc, char **argv) {
exit(1);
}
mdb_set_bind_size(mdb, bind_size);
/* read the catalog */
if (!mdb_read_catalog (mdb, MDB_ANY)) {
fprintf(stderr,"File does not appear to be an Access database\n");