From 948e63c5380a16f22534934c372119bef18ca10a Mon Sep 17 00:00:00 2001 From: Silvan Calarco Date: Sat, 12 Oct 2013 17:34:55 +0200 Subject: [PATCH] sqlite3-backend, distroquery: split database into dbs with names: -.db and -sources.db --- src/backend-sqlite3.c | 49 ++++++++----- src/distromatic.c | 33 ++------- src/distroquery.c | 151 ++++++++++++++++++++++++-------------- src/functions.c | 20 +++++ src/include/distromatic.h | 2 +- src/include/functions.h | 1 + 6 files changed, 153 insertions(+), 103 deletions(-) diff --git a/src/backend-sqlite3.c b/src/backend-sqlite3.c index 9e29cec..5f662b1 100644 --- a/src/backend-sqlite3.c +++ b/src/backend-sqlite3.c @@ -22,6 +22,7 @@ #include "headerlist.h" #include "changelog.h" #include "backend-sqlite3.h" +#include "functions.h" #define SQLITE_TABLE_files "id INTEGER PRIMARY KEY, "\ "name STRING, firstchild INTEGER, next INTEGER, parent INTEGER, numproviders INTEGER" @@ -186,20 +187,19 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db) { } -int generateSQLite_provided(struct configTag* ct, sqlite3 *db) { +int generateSQLite_provided(struct configTag* ct, sqlite3 *db, int arch) { int i; struct providedList* provided; - for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) { - snprintf(sqlite3_query, PATH_MAX, "provided_%s", ct->arch[i]); +// for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) { + snprintf(sqlite3_query, PATH_MAX, "provided"); SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_provided); SQLite_begin_transaction(db); - provided = ct->providedlist_idx[i][0]; + provided = ct->providedlist_idx[arch][0]; while (provided) { - snprintf(sqlite3_query, PATH_MAX, "INSERT INTO provided_%s VALUES(%d,?,%d,%d);", - ct->arch[i], + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO provided VALUES(%d,?,%d,%d);", provided->id, provided->flags, provided->numproviders); @@ -218,7 +218,7 @@ int generateSQLite_provided(struct configTag* ct, sqlite3 *db) { /* if (sqlite3_finalize(stmt)) { fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db)); }*/ - } +// } return 0; } @@ -399,16 +399,16 @@ generateSQLite_packages(struct configTag *ct, sqlite3 *db, int arch) { struct headerList* currpackage; int i; - snprintf(sqlite3_query, PATH_MAX, "packages_%s", ct->arch[arch]); + snprintf(sqlite3_query, PATH_MAX, "packages"); SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_packages); - snprintf(sqlite3_query, PATH_MAX, "packages_files_rel_%s", ct->arch[arch]); + snprintf(sqlite3_query, PATH_MAX, "packages_files_rel"); SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_packages_files_rel); SQLite_begin_transaction(db); currpackage = ct->headerlist[arch]; while (currpackage) { - snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_%s VALUES(NULL,?,%d,%d,?,?,?,?,?,%d,%d);", + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages VALUES(NULL,?,%d,%d,?,?,?,?,?,%d,%d);", ct->arch[arch], currpackage->altrepository, currpackage->epoch, @@ -434,7 +434,7 @@ generateSQLite_packages(struct configTag *ct, sqlite3 *db, int arch) { /* packages <-> files relations */ for (i = 0; i < currpackage->filenamecount; i++) { - snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel_%s VALUES(NULL,%d,%d);", + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%d,%d);", ct->arch[arch], currpackage->id, currpackage->file[i]->id); @@ -461,7 +461,23 @@ generateSQLite(struct configTag *ct) char dbname[PATH_MAX]; int i; - snprintf(dbname, PATH_MAX, "%sdistromatic.db", ct->repository_dir); + for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) { + snprintf(dbname, PATH_MAX, "%s%s-%s.db", ct->repository_dir, ct->tag, ct->arch[i]); + unlink(dbname); + + if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) { + logmsg(LOG_ERROR, "sqlite3_open_v2: %s %s", sqlite3_errmsg(db), dbname); + if (db) sqlite3_close(db); + return 1; + } + + generateSQLite_packages(ct, db, i); +// generateSQLite_files(ct, db); + generateSQLite_provided(ct, db, i); + sqlite3_close(db); + } + + snprintf(dbname, PATH_MAX, "%s%s-sources.db", ct->repository_dir, ct->tag); unlink(dbname); if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) { @@ -469,15 +485,8 @@ generateSQLite(struct configTag *ct) return 1; } - generateSQLite_packagers(db); -// generateSQLite_files(ct, db); - generateSQLite_provided(ct, db); - - for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) { - generateSQLite_packages(ct, db, i); - } - generateSQLite_sources(ct, db); + generateSQLite_packagers(db); sqlite3_close(db); return 0; diff --git a/src/distromatic.c b/src/distromatic.c index 2f08dbb..8576431 100644 --- a/src/distromatic.c +++ b/src/distromatic.c @@ -187,26 +187,6 @@ program_version(void) { printf("%s\n", freelicense[linenum]); } -/* Obtain a backtrace and print it to stdout. */ -void backtraceHandler(int sig) -{ - void *array[10]; - size_t size; - char **strings; - size_t i; - - size = backtrace (array, 10); - strings = (char**)backtrace_symbols (array, size); - - fprintf(stderr, "Obtained %zd stack frames.\n", size); - - for (i = 0; i < size; i++) - fprintf(stderr, "%s\n", strings[i]); - - free (strings); - exit(1); -} - /* * checks if given single requirement is met by given provides * @@ -508,7 +488,7 @@ resolveFirstLevelDependencies(struct configTag *ct, int archidx) provided->flags=0; } } - if (!scanheader) { + if (!scanheader && incremental_mode) { snprintf(warning,PATH_MAX,"%s(%s,%s): missing provider for %s", currheader->name, ct->arch[archidx], @@ -768,7 +748,7 @@ resolveFirstLevelSourceDependencies(struct configTag *ct, int archidx) logmsg(LOG_DEBUG,"%s is a build provider for %s",provided->name,currsourceheader->name); } } - if (provided->numbuildproviders == 0) { + if (provided->numbuildproviders == 0 && incremental_mode) { snprintf(warning,PATH_MAX,"missing build provider for %s",currsourceheader->require[i]->name); fprintf(stderr,"Warning: %s(source,%s): %s\n", currsourceheader->name, @@ -1086,7 +1066,7 @@ main(int argc, char *argv[]) recursive_mode = 0; incremental_mode = 0; obsolete_packages = 0; - resolve_dependencies = 0; + //resolve_dependencies = 0; name = NULL; } @@ -1221,9 +1201,12 @@ main(int argc, char *argv[]) } // if (genheader_mode) if (mode & MODE_SQLITE3) { - if (!quietmode) printf("Generating sqlite db...\n"); + if (!quietmode) printf("Generating sqlite databases...\n"); logmsg(LOG_DEBUG,"generateSQLite - start"); - generateSQLite(configtag); + if (generateSQLite(configtag)) { + logmsg(LOG_ERROR, "could not generate sqlite databases; aborting."); + exit(1); + } logmsg(LOG_DEBUG,"generateSQLite - done"); } diff --git a/src/distroquery.c b/src/distroquery.c index a698c81..6c1897e 100644 --- a/src/distroquery.c +++ b/src/distroquery.c @@ -71,15 +71,20 @@ #include "distromatic.h" #include "functions.h" +// must be as big as ARCHS_MAX (5) +const char* ARCHS[ARCHS_MAX] = { "i586" , "x86_64" , "arm" , "", ""}; + static struct configTag *firstconfigtag = NULL, *configtag = NULL; char *query; char *query_package; char *query_repository; char *query_arch; +int query_archs[ARCHS_MAX]; + struct configTag *query_repositories[100]; int search_milestone1 = 0, search_milestone2 = 1, search_devel = 1; -int search_arm = 0, search_x86_64 = 0, search_i586 = 1, search_sources = 0; +int search_sources = 0; /* Converts a hex character to its integer value */ char from_hex(char ch) { @@ -170,6 +175,15 @@ void printInputForm() { printf("]]>"); } +int find_query_arch(char* arch) { + int i; + + for (i = 0; i < ARCHS_MAX; i++) { + if (!strcmp(arch, ARCHS[i]) && query_archs[i]) return i; + } + return -1; +} + void printQueryData() { int a, i, j, k, numresults = 0; @@ -182,46 +196,54 @@ void printQueryData() { if (query[i] == ' ') query[i] = '%'; } - for (i = 0; query_repositories[i] != NULL; i++) { +/* for (i = 0; query_repositories[i] != NULL; i++) { + for (a = 0; a < ARCHS_MAX && query_repositories[i]->arch[a]; a++) { + if (find_query_arch(query_repositories[i]->arch[a]) >= 0) { snprintf(dbname, PATH_MAX, "%sdistromatic.db", query_repositories[i]->repository_dir); if (sqlite3_open_v2(dbname, (sqlite3**)&query_repositories[i]->db, SQLITE_OPEN_READONLY, NULL)) { if (query_repositories[i]->db) sqlite3_close(query_repositories[i]->db); query_repositories[i]->db = NULL; fprintf(stderr, "ERROR: unable to open sqlite3 db %s; aborting.\n", dbname); } - } + }*/ for (i = 0; query_repositories[i] != NULL; i++) { for (a = 0; a < ARCHS_MAX && query_repositories[i]->arch[a]; a++) { - if ((search_i586 && !strcmp(query_repositories[i]->arch[a],"i586")) || - (search_arm && !strcmp(query_repositories[i]->arch[a],"arm")) || - (search_x86_64 && !strcmp(query_repositories[i]->arch[a],"x86_64"))) { - snprintf(sql, PATH_MAX, - "SELECT * FROM packages_%s WHERE name LIKE '%%%s%%' OR summary LIKE '%%%s%%' OR description LIKE '%%%s%%' OR version LIKE '%%%s%%' " - "ORDER BY name = '%s' DESC, name LIKE '%s%%' DESC, name LIKE '%%%s%%' DESC, " - "groupdescr LIKE 'Graphical Desktop/%%' DESC, groupdescr LIKE 'Applications/%%' DESC, " - "groupdescr LIKE 'Development/%%' ASC, groupdescr LIKE 'Documentation%%' ASC, groupdescr LIKE 'System/Libraries%%' ASC, " - "name LIKE '%%-devel' ASC, name LIKE '%%-debug' ASC " - "LIMIT 100", - query_repositories[i]->arch[a], query, query, query, query, query, query); - if (query_repositories[i]->db && - (sqlite3_prepare_v2((sqlite3*)query_repositories[i]->db, sql, strlen(sql), &statement, NULL) == SQLITE_OK)) { - while (sqlite3_step(statement) == SQLITE_ROW) { - numresults++; - printf("

 %s 
 " - "
 %s 
 " - "
 %s 

" - "%s %s-%s
%s", - query_repositories[i]->arch[a], - query_repositories[i]->tag, - sqlite3_column_text(statement,7), - sqlite3_column_text(statement,1), - sqlite3_column_text(statement,4), - sqlite3_column_text(statement,5), - sqlite3_column_text(statement,6)); + if (find_query_arch(query_repositories[i]->arch[a]) >= 0) { + if (!query_repositories[i]->db[a]) { + snprintf(dbname, PATH_MAX, "%s%s-%s.db", + query_repositories[i]->repository_dir, query_repositories[i]->tag, query_repositories[i]->arch[a]); + if (sqlite3_open_v2(dbname, (sqlite3**)&query_repositories[i]->db[a], SQLITE_OPEN_READONLY, NULL)) { + if (query_repositories[i]->db) sqlite3_close(query_repositories[i]->db[a]); + query_repositories[i]->db[a] = NULL; + fprintf(stderr, "ERROR: unable to open sqlite3 db %s; ignoring.\n", dbname); + } + } + if (query_repositories[i]->db[a]) { + snprintf(sql, PATH_MAX, + "SELECT * FROM packages WHERE name LIKE '%%%s%%' OR summary LIKE '%%%s%%' OR description LIKE '%%%s%%' OR version LIKE '%%%s%%' " + "ORDER BY name = '%s' DESC, name LIKE '%s%%' DESC, name LIKE '%%%s%%' DESC, " + "groupdescr LIKE 'Graphical Desktop/%%' DESC, groupdescr LIKE 'Applications/%%' DESC, " + "groupdescr LIKE 'Development/%%' ASC, groupdescr LIKE 'Documentation%%' ASC, groupdescr LIKE 'System/Libraries%%' ASC, " + "name LIKE '%%-devel' ASC, name LIKE '%%-debug' ASC " + "LIMIT 100", query, query, query, query, query, query); + if (sqlite3_prepare_v2((sqlite3*)query_repositories[i]->db[a], sql, strlen(sql), &statement, NULL) == SQLITE_OK) { + while (sqlite3_step(statement) == SQLITE_ROW) { + numresults++; + printf("

 %s 
 " + "
 %s 
 " + "
 %s 

" + "%s %s-%s
%s", + query_repositories[i]->arch[a], + query_repositories[i]->tag, + sqlite3_column_text(statement,7), + sqlite3_column_text(statement,1), + sqlite3_column_text(statement,4), + sqlite3_column_text(statement,5), + sqlite3_column_text(statement,6)); - printf("

%s
", - sqlite3_column_text(statement,8)); + printf("

%s
", + sqlite3_column_text(statement,8)); /* install */ /* printf("
" @@ -233,7 +255,7 @@ void printQueryData() { sqlite3_column_text(statement,1));*/ /* download */ - printf("
" + printf("", query_repositories[i]->download_prefix, query_repositories[i]->download_dir, @@ -241,18 +263,18 @@ void printQueryData() { sqlite3_column_text(statement,4), sqlite3_column_text(statement,5)); /* details */ - printf(" ", - query_repositories[i]->tag, - sqlite3_column_text(statement,1), - query_repositories[i]->arch[a]); + printf(" ", + query_repositories[i]->tag, + sqlite3_column_text(statement,1), + query_repositories[i]->arch[a]); - printf("

"); + printf("

"); + } } } } - printf("\n"); } } @@ -262,8 +284,16 @@ void printQueryData() { "ORDER BY name = '%s' DESC, name LIKE '%s%%' DESC, name LIKE '%%%s%%' DESC LIMIT 100", query, query, query, query, query, query); for (i = 0; query_repositories[i] != NULL; i++) { - if (query_repositories[i]->db && - (sqlite3_prepare_v2((sqlite3*)query_repositories[i]->db, sql, strlen(sql), &statement, NULL) == SQLITE_OK)) { + if (!query_repositories[i]->db[ARCHS_MAX]) { + snprintf(dbname, PATH_MAX, "%s%s-sources.db", query_repositories[i]->repository_dir, query_repositories[i]->tag); + if (sqlite3_open_v2(dbname, (sqlite3**)&query_repositories[i]->db[ARCHS_MAX], SQLITE_OPEN_READONLY, NULL)) { + if (query_repositories[i]->db) sqlite3_close(query_repositories[i]->db[a]); + query_repositories[i]->db[ARCHS_MAX] = NULL; + fprintf(stderr, "ERROR: unable to open sqlite3 db %s; ignoring.\n", dbname); + } + } + if (query_repositories[i]->db[ARCHS_MAX] && + (sqlite3_prepare_v2((sqlite3*)query_repositories[i]->db[ARCHS_MAX], sql, strlen(sql), &statement, NULL) == SQLITE_OK)) { while (sqlite3_step(statement) == SQLITE_ROW) { numresults++; printf("

 Source 
 " @@ -282,7 +312,6 @@ void printQueryData() { sqlite3_column_text(statement,9)); } } - printf("\n"); } } printf("
"); @@ -291,8 +320,12 @@ void printQueryData() { printf("", numresults); for (i = 0; query_repositories[i] != NULL; i++) { - sqlite3_close((sqlite3*)query_repositories[i]->db); - query_repositories[i]->db = NULL; + for (a = 0; a <= ARCHS_MAX && query_repositories[i]->arch[a]; a++) { + if (query_repositories[i]->db[a]) { + sqlite3_close((sqlite3*)query_repositories[i]->db[a]); + query_repositories[i]->db[a] = NULL; + } + } } } @@ -300,23 +333,24 @@ void printPackageData() { int a, i, j, k, numresults = 0; char dbname[PATH_MAX]; + sqlite3 *db; sqlite3_stmt *statement; char sql[PATH_MAX]; struct configTag* ct = findRepositoryByTag(query_repository); printf("repository_dir); - if (sqlite3_open_v2(dbname, (sqlite3**)&ct->db, SQLITE_OPEN_READONLY, NULL)) { - if (ct->db) sqlite3_close(ct->db); - ct->db = NULL; + snprintf(dbname, PATH_MAX, "%s%s-%s.db", ct->repository_dir, ct->tag, query_arch); + if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READONLY, NULL)) { + if (db) sqlite3_close(db); + db = NULL; fprintf(stderr, "ERROR: unable to open sqlite3 db %s; aborting.\n", dbname); return; } - snprintf(sql, PATH_MAX, "SELECT * FROM packages_%s WHERE name = '%s'", query_arch, query_package); - if (ct->db && - (sqlite3_prepare_v2((sqlite3*)ct->db, sql, strlen(sql), &statement, NULL) == SQLITE_OK)) { + snprintf(sql, PATH_MAX, "SELECT * FROM packages WHERE name = '%s'", query_package); + if (db && + (sqlite3_prepare_v2(db, sql, strlen(sql), &statement, NULL) == SQLITE_OK)) { while (sqlite3_step(statement) == SQLITE_ROW) { numresults++; printf("

%s
 " @@ -364,8 +398,8 @@ void printPackageData() { printf("

%d result(s) found.", numresults);*/ printf("]]>"); - sqlite3_close(ct->db); - ct->db = NULL; + sqlite3_close(db); + db = NULL; } void parse_request_variables(char *data) { @@ -393,11 +427,11 @@ void parse_request_variables(char *data) { } else if (!strcmp(vartok, "search_devel")) { search_devel = (strstr(valuetok, "false") != valuetok); } else if (!strcmp(vartok, "search_i586")) { - search_i586 = (strstr(valuetok, "false") != valuetok); + query_archs[0] = strstr(valuetok, "false") != valuetok; } else if (!strcmp(vartok, "search_x86_64")) { - search_x86_64 = (strstr(valuetok, "false") != valuetok); + query_archs[1] = strstr(valuetok, "false") != valuetok; } else if (!strcmp(vartok, "search_arm")) { - search_arm = (strstr(valuetok, "false") != valuetok); + query_archs[2] = strstr(valuetok, "false") != valuetok; } else if (!strcmp(vartok, "search_sources")) { search_sources = (strstr(valuetok, "false") != valuetok); } @@ -429,6 +463,9 @@ main(int argc, char *argv[]) // char *configfile = NULL; char *data; + // install backtrace handler + signal(SIGSEGV, backtraceHandler); + printf("Content-Type: text/xml;charset=utf-8\n\n"); firstconfigtag = read_configuration(DEFAULT_CONFIGFILE); diff --git a/src/functions.c b/src/functions.c index 28b74f6..37c4a93 100644 --- a/src/functions.c +++ b/src/functions.c @@ -77,6 +77,26 @@ static int debug_log = 0; static struct configDefaults configdefaults; static struct configTag *firstconfigtag = NULL; +/* Obtain a backtrace and print it to stdout. */ +void backtraceHandler(int sig) +{ + void *array[10]; + size_t size; + char **strings; + size_t i; + + size = backtrace (array, 10); + strings = (char**)backtrace_symbols (array, size); + + fprintf(stderr, "Obtained %zd stack frames.\n", size); + + for (i = 0; i < size; i++) + fprintf(stderr, "%s\n", strings[i]); + + free (strings); + exit(1); +} + struct configTag* findRepositoryByTag(const char *tag) { struct configTag *ct = firstconfigtag; diff --git a/src/include/distromatic.h b/src/include/distromatic.h index 57cc1c5..6856ce5 100644 --- a/src/include/distromatic.h +++ b/src/include/distromatic.h @@ -39,7 +39,7 @@ struct configTag { struct fileTree *filetree[ARCHS_MAX]; struct headerStats stats; struct configTag *next; - void *db; + void *db[ARCHS_MAX + 1]; }; #endif diff --git a/src/include/functions.h b/src/include/functions.h index 82e327d..646a70e 100644 --- a/src/include/functions.h +++ b/src/include/functions.h @@ -38,5 +38,6 @@ char *htmlcleanNoBr(char *source,char *dest,unsigned int max); int scansdir(const char *dir, struct dirent ***namelist, int (*select)(const struct dirent *), int (*compar)(const struct dirent **, const struct dirent **)); +void backtraceHandler(int sig); #endif /* FUNCTIONS_H */