openmamba/distromatic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

sqlite3-backend, distroquery: split database into dbs with names: <repository>-<arch>.db and <repository>-sources.db

Browse Source
This commit is contained in:
Silvan Calarco 2013-10-12 17:34:55 +02:00
parent b8b1a3db49
commit 948e63c538
6 changed files with 153 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
src/backend-sqlite3.c
View File

@ -22,6 +22,7 @@
#include "headerlist.h"
#include "changelog.h"
#include "backend-sqlite3.h"
#include "functions.h"
#define SQLITE_TABLE_files "id INTEGER PRIMARY KEY, "\
"name STRING, firstchild INTEGER, next INTEGER, parent INTEGER, numproviders INTEGER"
@ -186,20 +187,19 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db) {
}
int generateSQLite_provided(struct configTag* ct, sqlite3 *db) {
int generateSQLite_provided(struct configTag* ct, sqlite3 *db, int arch) {
int i;
struct providedList* provided;
for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) {
snprintf(sqlite3_query, PATH_MAX, "provided_%s", ct->arch[i]);
// for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) {
snprintf(sqlite3_query, PATH_MAX, "provided");
SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_provided);
SQLite_begin_transaction(db);
provided = ct->providedlist_idx[i][0];
provided = ct->providedlist_idx[arch][0];
while (provided) {
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO provided_%s VALUES(%d,?,%d,%d);",
ct->arch[i],
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO provided VALUES(%d,?,%d,%d);",
provided->id,
provided->flags,
provided->numproviders);
@ -218,7 +218,7 @@ int generateSQLite_provided(struct configTag* ct, sqlite3 *db) {
/* if (sqlite3_finalize(stmt)) {
fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db));
}*/
}
// }
return 0;
}
@ -399,16 +399,16 @@ generateSQLite_packages(struct configTag *ct, sqlite3 *db, int arch) {
struct headerList* currpackage;
int i;
snprintf(sqlite3_query, PATH_MAX, "packages_%s", ct->arch[arch]);
snprintf(sqlite3_query, PATH_MAX, "packages");
SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_packages);
snprintf(sqlite3_query, PATH_MAX, "packages_files_rel_%s", ct->arch[arch]);
snprintf(sqlite3_query, PATH_MAX, "packages_files_rel");
SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_packages_files_rel);
SQLite_begin_transaction(db);
currpackage = ct->headerlist[arch];
while (currpackage) {
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_%s VALUES(NULL,?,%d,%d,?,?,?,?,?,%d,%d);",
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages VALUES(NULL,?,%d,%d,?,?,?,?,?,%d,%d);",
ct->arch[arch],
currpackage->altrepository,
currpackage->epoch,
@ -434,7 +434,7 @@ generateSQLite_packages(struct configTag *ct, sqlite3 *db, int arch) {
/* packages <-> files relations */
for (i = 0; i < currpackage->filenamecount; i++) {
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel_%s VALUES(NULL,%d,%d);",
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%d,%d);",
ct->arch[arch],
currpackage->id,
currpackage->file[i]->id);
@ -461,7 +461,23 @@ generateSQLite(struct configTag *ct)
char dbname[PATH_MAX];
int i;
snprintf(dbname, PATH_MAX, "%sdistromatic.db", ct->repository_dir);
for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) {
snprintf(dbname, PATH_MAX, "%s%s-%s.db", ct->repository_dir, ct->tag, ct->arch[i]);
unlink(dbname);
if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) {
logmsg(LOG_ERROR, "sqlite3_open_v2: %s %s", sqlite3_errmsg(db), dbname);
if (db) sqlite3_close(db);
return 1;
}
generateSQLite_packages(ct, db, i);
// generateSQLite_files(ct, db);
generateSQLite_provided(ct, db, i);
sqlite3_close(db);
}
snprintf(dbname, PATH_MAX, "%s%s-sources.db", ct->repository_dir, ct->tag);
unlink(dbname);
if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) {
@ -469,15 +485,8 @@ generateSQLite(struct configTag *ct)
return 1;
}
generateSQLite_packagers(db);
// generateSQLite_files(ct, db);
generateSQLite_provided(ct, db);
for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) {
generateSQLite_packages(ct, db, i);
}
generateSQLite_sources(ct, db);
generateSQLite_packagers(db);
sqlite3_close(db);
return 0;

33
src/distromatic.c
View File

@ -187,26 +187,6 @@ program_version(void) {
printf("%s\n", freelicense[linenum]);
}
/* Obtain a backtrace and print it to stdout. */
void backtraceHandler(int sig)
{
void *array[10];
size_t size;
char **strings;
size_t i;
size = backtrace (array, 10);
strings = (char**)backtrace_symbols (array, size);
fprintf(stderr, "Obtained %zd stack frames.\n", size);
for (i = 0; i < size; i++)
fprintf(stderr, "%s\n", strings[i]);
free (strings);
exit(1);
}
/*
* checks if given single requirement is met by given provides
*
@ -508,7 +488,7 @@ resolveFirstLevelDependencies(struct configTag *ct, int archidx)
provided->flags=0;
}
}
if (!scanheader) {
if (!scanheader && incremental_mode) {
snprintf(warning,PATH_MAX,"%s(%s,%s): missing provider for %s",
currheader->name,
ct->arch[archidx],
@ -768,7 +748,7 @@ resolveFirstLevelSourceDependencies(struct configTag *ct, int archidx)
logmsg(LOG_DEBUG,"%s is a build provider for %s",provided->name,currsourceheader->name);
}
}
if (provided->numbuildproviders == 0) {
if (provided->numbuildproviders == 0 && incremental_mode) {
snprintf(warning,PATH_MAX,"missing build provider for %s",currsourceheader->require[i]->name);
fprintf(stderr,"Warning: %s(source,%s): %s\n",
currsourceheader->name,
@ -1086,7 +1066,7 @@ main(int argc, char *argv[])
recursive_mode = 0;
incremental_mode = 0;
obsolete_packages = 0;
resolve_dependencies = 0;
//resolve_dependencies = 0;
name = NULL;
}
@ -1221,9 +1201,12 @@ main(int argc, char *argv[])
} // if (genheader_mode)
if (mode & MODE_SQLITE3) {
if (!quietmode) printf("Generating sqlite db...\n");
if (!quietmode) printf("Generating sqlite databases...\n");
logmsg(LOG_DEBUG,"generateSQLite - start");
generateSQLite(configtag);
if (generateSQLite(configtag)) {
logmsg(LOG_ERROR, "could not generate sqlite databases; aborting.");
exit(1);
}
logmsg(LOG_DEBUG,"generateSQLite - done");
}

151
src/distroquery.c
View File

@ -71,15 +71,20 @@
#include "distromatic.h"
#include "functions.h"
// must be as big as ARCHS_MAX (5)
const char* ARCHS[ARCHS_MAX] = { "i586" , "x86_64" , "arm" , "", ""};
static struct configTag *firstconfigtag = NULL, *configtag = NULL;
char *query;
char *query_package;
char *query_repository;
char *query_arch;
int query_archs[ARCHS_MAX];
struct configTag *query_repositories[100];
int search_milestone1 = 0, search_milestone2 = 1, search_devel = 1;
int search_arm = 0, search_x86_64 = 0, search_i586 = 1, search_sources = 0;
int search_sources = 0;
/* Converts a hex character to its integer value */
char from_hex(char ch) {
@ -170,6 +175,15 @@ void printInputForm() {
printf("]]></queryform>");
}
int find_query_arch(char* arch) {
int i;
for (i = 0; i < ARCHS_MAX; i++) {
if (!strcmp(arch, ARCHS[i]) && query_archs[i]) return i;
}
return -1;
}
void printQueryData() {
int a, i, j, k, numresults = 0;
@ -182,46 +196,54 @@ void printQueryData() {
if (query[i] == ' ') query[i] = '%';
}
for (i = 0; query_repositories[i] != NULL; i++) {
/* for (i = 0; query_repositories[i] != NULL; i++) {
for (a = 0; a < ARCHS_MAX && query_repositories[i]->arch[a]; a++) {
if (find_query_arch(query_repositories[i]->arch[a]) >= 0) {
snprintf(dbname, PATH_MAX, "%sdistromatic.db", query_repositories[i]->repository_dir);
if (sqlite3_open_v2(dbname, (sqlite3**)&query_repositories[i]->db, SQLITE_OPEN_READONLY, NULL)) {
if (query_repositories[i]->db) sqlite3_close(query_repositories[i]->db);
query_repositories[i]->db = NULL;
fprintf(stderr, "ERROR: unable to open sqlite3 db %s; aborting.\n", dbname);
}
}
}*/
for (i = 0; query_repositories[i] != NULL; i++) {
for (a = 0; a < ARCHS_MAX && query_repositories[i]->arch[a]; a++) {
if ((search_i586 && !strcmp(query_repositories[i]->arch[a],"i586")) ||
(search_arm && !strcmp(query_repositories[i]->arch[a],"arm")) ||
(search_x86_64 && !strcmp(query_repositories[i]->arch[a],"x86_64"))) {
snprintf(sql, PATH_MAX,
"SELECT * FROM packages_%s WHERE name LIKE '%%%s%%' OR summary LIKE '%%%s%%' OR description LIKE '%%%s%%' OR version LIKE '%%%s%%' "
"ORDER BY name = '%s' DESC, name LIKE '%s%%' DESC, name LIKE '%%%s%%' DESC, "
"groupdescr LIKE 'Graphical Desktop/%%' DESC, groupdescr LIKE 'Applications/%%' DESC, "
"groupdescr LIKE 'Development/%%' ASC, groupdescr LIKE 'Documentation%%' ASC, groupdescr LIKE 'System/Libraries%%' ASC, "
"name LIKE '%%-devel' ASC, name LIKE '%%-debug' ASC "
"LIMIT 100",
query_repositories[i]->arch[a], query, query, query, query, query, query);
if (query_repositories[i]->db &&
(sqlite3_prepare_v2((sqlite3*)query_repositories[i]->db, sql, strlen(sql), &statement, NULL) == SQLITE_OK)) {
while (sqlite3_step(statement) == SQLITE_ROW) {
numresults++;
printf("<div style='position:relative'><hr><br><div style='display:inline;color:white;background-color:green'>&nbsp;%s&nbsp;</div>&nbsp;"
"<div style='display:inline;color:black;background-color:yellow'>&nbsp;%s&nbsp;</div>&nbsp;"
"<div style='display:inline;color:black;background-color:lightblue'>&nbsp;%s&nbsp;</div><br>"
"<b>%s</b> %s-%s<br>%s",
query_repositories[i]->arch[a],
query_repositories[i]->tag,
sqlite3_column_text(statement,7),
sqlite3_column_text(statement,1),
sqlite3_column_text(statement,4),
sqlite3_column_text(statement,5),
sqlite3_column_text(statement,6));
if (find_query_arch(query_repositories[i]->arch[a]) >= 0) {
if (!query_repositories[i]->db[a]) {
snprintf(dbname, PATH_MAX, "%s%s-%s.db",
query_repositories[i]->repository_dir, query_repositories[i]->tag, query_repositories[i]->arch[a]);
if (sqlite3_open_v2(dbname, (sqlite3**)&query_repositories[i]->db[a], SQLITE_OPEN_READONLY, NULL)) {
if (query_repositories[i]->db) sqlite3_close(query_repositories[i]->db[a]);
query_repositories[i]->db[a] = NULL;
fprintf(stderr, "ERROR: unable to open sqlite3 db %s; ignoring.\n", dbname);
}
}
if (query_repositories[i]->db[a]) {
snprintf(sql, PATH_MAX,
"SELECT * FROM packages WHERE name LIKE '%%%s%%' OR summary LIKE '%%%s%%' OR description LIKE '%%%s%%' OR version LIKE '%%%s%%' "
"ORDER BY name = '%s' DESC, name LIKE '%s%%' DESC, name LIKE '%%%s%%' DESC, "
"groupdescr LIKE 'Graphical Desktop/%%' DESC, groupdescr LIKE 'Applications/%%' DESC, "
"groupdescr LIKE 'Development/%%' ASC, groupdescr LIKE 'Documentation%%' ASC, groupdescr LIKE 'System/Libraries%%' ASC, "
"name LIKE '%%-devel' ASC, name LIKE '%%-debug' ASC "
"LIMIT 100", query, query, query, query, query, query);
if (sqlite3_prepare_v2((sqlite3*)query_repositories[i]->db[a], sql, strlen(sql), &statement, NULL) == SQLITE_OK) {
while (sqlite3_step(statement) == SQLITE_ROW) {
numresults++;
printf("<div style='position:relative'><hr><br><div style='display:inline;color:white;background-color:green'>&nbsp;%s&nbsp;</div>&nbsp;"
"<div style='display:inline;color:black;background-color:yellow'>&nbsp;%s&nbsp;</div>&nbsp;"
"<div style='display:inline;color:black;background-color:lightblue'>&nbsp;%s&nbsp;</div><br>"
"<b>%s</b> %s-%s<br>%s",
query_repositories[i]->arch[a],
query_repositories[i]->tag,
sqlite3_column_text(statement,7),
sqlite3_column_text(statement,1),
sqlite3_column_text(statement,4),
sqlite3_column_text(statement,5),
sqlite3_column_text(statement,6));
printf("<br><br><div style='text-align:justify;text-justify:inter-word;font-style:italic'>%s</div>",
sqlite3_column_text(statement,8));
printf("<br><br><div style='text-align:justify;text-justify:inter-word;font-style:italic'>%s</div>",
sqlite3_column_text(statement,8));
/* install */
/* printf("<div align=right style='position:absolute;top:5px;right:0;padding:5px;'>"
@ -233,7 +255,7 @@ void printQueryData() {
sqlite3_column_text(statement,1));*/
/* download */
printf("<div style='display:inline;background-color:green'>"
printf("<div style='display:inline;background-color:green'>"
"<a href=\"%s%s/SRPMS.base/%s-%s-%s.src.rpm\" style=\"color:white\">&nbsp;Download&nbsp;</a></div>",
query_repositories[i]->download_prefix,
query_repositories[i]->download_dir,
@ -241,18 +263,18 @@ void printQueryData() {
sqlite3_column_text(statement,4),
sqlite3_column_text(statement,5));
/* details */
printf("&nbsp;<div style='display:inline;background-color:lightblue;font-weight:strong'>"
"<a href='javascript:distroquery_request("
"\"repository=%s&package=%s&arch=%s\")' style=\"color:black\">&nbsp;Details&nbsp;</a></div>",
query_repositories[i]->tag,
sqlite3_column_text(statement,1),
query_repositories[i]->arch[a]);
printf("&nbsp;<div style='display:inline;background-color:lightblue;font-weight:strong'>"
"<a href='javascript:distroquery_request("
"\"repository=%s&package=%s&arch=%s\")' style=\"color:black\">&nbsp;Details&nbsp;</a></div>",
query_repositories[i]->tag,
sqlite3_column_text(statement,1),
query_repositories[i]->arch[a]);
printf("</div></div><br>");
printf("</div></div><br>");
}
}
}
}
printf("\n");
}
}
@ -262,8 +284,16 @@ void printQueryData() {
"ORDER BY name = '%s' DESC, name LIKE '%s%%' DESC, name LIKE '%%%s%%' DESC LIMIT 100",
query, query, query, query, query, query);
for (i = 0; query_repositories[i] != NULL; i++) {
if (query_repositories[i]->db &&
(sqlite3_prepare_v2((sqlite3*)query_repositories[i]->db, sql, strlen(sql), &statement, NULL) == SQLITE_OK)) {
if (!query_repositories[i]->db[ARCHS_MAX]) {
snprintf(dbname, PATH_MAX, "%s%s-sources.db", query_repositories[i]->repository_dir, query_repositories[i]->tag);
if (sqlite3_open_v2(dbname, (sqlite3**)&query_repositories[i]->db[ARCHS_MAX], SQLITE_OPEN_READONLY, NULL)) {
if (query_repositories[i]->db) sqlite3_close(query_repositories[i]->db[a]);
query_repositories[i]->db[ARCHS_MAX] = NULL;
fprintf(stderr, "ERROR: unable to open sqlite3 db %s; ignoring.\n", dbname);
}
}
if (query_repositories[i]->db[ARCHS_MAX] &&
(sqlite3_prepare_v2((sqlite3*)query_repositories[i]->db[ARCHS_MAX], sql, strlen(sql), &statement, NULL) == SQLITE_OK)) {
while (sqlite3_step(statement) == SQLITE_ROW) {
numresults++;
printf("<hr><br><div style='display:inline;color:black;background-color:red'>&nbsp;Source&nbsp;</div>&nbsp;"
@ -282,7 +312,6 @@ void printQueryData() {
sqlite3_column_text(statement,9));
}
}
printf("\n");
}
}
printf("<hr>");
@ -291,8 +320,12 @@ void printQueryData() {
printf("<querystatus><![CDATA[%d result(s) shown.]]></querystatus>", numresults);
for (i = 0; query_repositories[i] != NULL; i++) {
sqlite3_close((sqlite3*)query_repositories[i]->db);
query_repositories[i]->db = NULL;
for (a = 0; a <= ARCHS_MAX && query_repositories[i]->arch[a]; a++) {
if (query_repositories[i]->db[a]) {
sqlite3_close((sqlite3*)query_repositories[i]->db[a]);
query_repositories[i]->db[a] = NULL;
}
}
}
}
@ -300,23 +333,24 @@ void printPackageData() {
int a, i, j, k, numresults = 0;
char dbname[PATH_MAX];
sqlite3 *db;
sqlite3_stmt *statement;
char sql[PATH_MAX];
struct configTag* ct = findRepositoryByTag(query_repository);
printf("<queryreply><![CDATA[");
snprintf(dbname, PATH_MAX, "%sdistromatic.db", ct->repository_dir);
if (sqlite3_open_v2(dbname, (sqlite3**)&ct->db, SQLITE_OPEN_READONLY, NULL)) {
if (ct->db) sqlite3_close(ct->db);
ct->db = NULL;
snprintf(dbname, PATH_MAX, "%s%s-%s.db", ct->repository_dir, ct->tag, query_arch);
if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READONLY, NULL)) {
if (db) sqlite3_close(db);
db = NULL;
fprintf(stderr, "ERROR: unable to open sqlite3 db %s; aborting.\n", dbname);
return;
}
snprintf(sql, PATH_MAX, "SELECT * FROM packages_%s WHERE name = '%s'", query_arch, query_package);
if (ct->db &&
(sqlite3_prepare_v2((sqlite3*)ct->db, sql, strlen(sql), &statement, NULL) == SQLITE_OK)) {
snprintf(sql, PATH_MAX, "SELECT * FROM packages WHERE name = '%s'", query_package);
if (db &&
(sqlite3_prepare_v2(db, sql, strlen(sql), &statement, NULL) == SQLITE_OK)) {
while (sqlite3_step(statement) == SQLITE_ROW) {
numresults++;
printf("<hr><br><div style='display:inline;color:white;background-color:green'> %s </div>&nbsp;"
@ -364,8 +398,8 @@ void printPackageData() {
printf("<hr><br>%d result(s) found.", numresults);*/
printf("]]></queryreply>");
sqlite3_close(ct->db);
ct->db = NULL;
sqlite3_close(db);
db = NULL;
}
void parse_request_variables(char *data) {
@ -393,11 +427,11 @@ void parse_request_variables(char *data) {
} else if (!strcmp(vartok, "search_devel")) {
search_devel = (strstr(valuetok, "false") != valuetok);
} else if (!strcmp(vartok, "search_i586")) {
search_i586 = (strstr(valuetok, "false") != valuetok);
query_archs[0] = strstr(valuetok, "false") != valuetok;
} else if (!strcmp(vartok, "search_x86_64")) {
search_x86_64 = (strstr(valuetok, "false") != valuetok);
query_archs[1] = strstr(valuetok, "false") != valuetok;
} else if (!strcmp(vartok, "search_arm")) {
search_arm = (strstr(valuetok, "false") != valuetok);
query_archs[2] = strstr(valuetok, "false") != valuetok;
} else if (!strcmp(vartok, "search_sources")) {
search_sources = (strstr(valuetok, "false") != valuetok);
}
@ -429,6 +463,9 @@ main(int argc, char *argv[])
// char *configfile = NULL;
char *data;
// install backtrace handler
signal(SIGSEGV, backtraceHandler);
printf("Content-Type: text/xml;charset=utf-8\n\n<distroquery>");
firstconfigtag = read_configuration(DEFAULT_CONFIGFILE);

20
src/functions.c
View File

@ -77,6 +77,26 @@ static int debug_log = 0;
static struct configDefaults configdefaults;
static struct configTag *firstconfigtag = NULL;
/* Obtain a backtrace and print it to stdout. */
void backtraceHandler(int sig)
{
void *array[10];
size_t size;
char **strings;
size_t i;
size = backtrace (array, 10);
strings = (char**)backtrace_symbols (array, size);
fprintf(stderr, "Obtained %zd stack frames.\n", size);
for (i = 0; i < size; i++)
fprintf(stderr, "%s\n", strings[i]);
free (strings);
exit(1);
}
struct configTag* findRepositoryByTag(const char *tag)
{
struct configTag *ct = firstconfigtag;

2
src/include/distromatic.h
View File

@ -39,7 +39,7 @@ struct configTag {
struct fileTree *filetree[ARCHS_MAX];
struct headerStats stats;
struct configTag *next;
void *db;
void *db[ARCHS_MAX + 1];
};
#endif

1
src/include/functions.h
View File

@ -38,5 +38,6 @@ char *htmlcleanNoBr(char *source,char *dest,unsigned int max);
int scansdir(const char *dir, struct dirent ***namelist,
int (*select)(const struct dirent *),
int (*compar)(const struct dirent **, const struct dirent **));
void backtraceHandler(int sig);
#endif /* FUNCTIONS_H */