openmamba/distromatic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

backend-sqlite3: add --gensqlite3files option to generate <repository>-<arch>-files.db databases

Browse Source
This commit is contained in:
Silvan Calarco 2013-10-14 16:22:21 +02:00
parent 2b99d7d46f
commit ec0c093578
2 changed files with 69 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
src/backend-sqlite3.c
View File

@ -26,6 +26,8 @@
char sqlite3_query[PATH_MAX]; char sqlite3_query[PATH_MAX];
sqlite3_stmt *stmt; sqlite3_stmt *stmt;
#define sqlite3_transaction_maxsize 10485760
long sqlite3_transaction_size; long sqlite3_transaction_size;
int SQLite_init_table(sqlite3 *db, const char* table_name, const char* table_query) { int SQLite_init_table(sqlite3 *db, const char* table_name, const char* table_query) {
@ -61,46 +63,44 @@ int SQLite_commit_transaction(sqlite3 *db) {
return 0; return 0;
} }
void SQLite_print_contents_subtree(sqlite3 *db, int SQLite_print_contents_subtree(sqlite3 *db,
struct fileTree* ft, struct fileTree* ft,
struct configTag* ct, struct configTag* ct,
int arch) { int arch) {
while (ft) { while (ft) {
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO files_%s VALUES(%d,?,%d,%d,%d,%d);", snprintf(sqlite3_query, PATH_MAX, "INSERT INTO files VALUES(%d,?,%d,%d,%d,%d);",
ct->arch[arch],
ft->id, ft->id,
(ft->firstchild?ft->firstchild->id:-1), (ft->firstchild?ft->firstchild->id:-1),
(ft->next?ft->next->id:-1), (ft->next?ft->next->id:-1),
(ft->parent?ft->parent->id:-1), (ft->parent?ft->parent->id:-1),
ft->numproviders); ft->numproviders);
sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL); sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL);
sqlite3_bind_text(stmt, 1, ft->name, -1, SQLITE_STATIC); sqlite3_bind_text(stmt, 1, ft->name, -1, SQLITE_STATIC);
if (sqlite3_step(stmt) != SQLITE_DONE) { if (sqlite3_step(stmt) != SQLITE_DONE) {
fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); fprintf(stderr, "ERROR: SQLite: %s (%s)\n", sqlite3_errmsg(db), sqlite3_query);
} }
/* sqlite3_transaction_size += strlen(sqlite3_query); sqlite3_transaction_size += strlen(sqlite3_query);
if (sqlite3_transaction_size >= 655360) { if (sqlite3_transaction_size >= sqlite3_transaction_maxsize) {
fprintf(stderr, "Commit: %ld\n", sqlite3_transaction_size); SQLite_commit_transaction(db);
sqlite3_exec(db, "COMMIT;", 0, 0, 0); SQLite_begin_transaction(db);
sqlite3_transaction_size = 0; }
}*/
if (ft->firstchild) { if (ft->firstchild) {
SQLite_print_contents_subtree(db, ft->firstchild, ct, arch); if (SQLite_print_contents_subtree(db, ft->firstchild, ct, arch)) return 1;
} }
ft=ft->next; ft=ft->next;
} }
return 0;
} }
#define SQLITE_TABLE_files "id INTEGER PRIMARY KEY, "\ #define SQLITE_TABLE_files "id INTEGER PRIMARY KEY, "\
"name STRING, firstchild INTEGER, next INTEGER, parent INTEGER, numproviders INTEGER" "name STRING, firstchild INTEGER, next INTEGER, parent INTEGER, numproviders INTEGER"
/* struct headerList **provider; */ /* struct headerList **provider; */
#define SQLITE_TABLE_packages_files_rel "id INTEGER PRIMARY KEY, id_package INTEGER, id_file INTEGER" #define SQLITE_TABLE_packages_files_rel "id INTEGER PRIMARY KEY, id_package INTEGER, id_file INTEGER, name STRING, version STRING, release STRING"
int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) { int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) {
@ -114,9 +114,8 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) {
SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_files); SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_files);
SQLite_begin_transaction(db); SQLite_begin_transaction(db);
SQLite_print_contents_subtree(db, ct->filetree[arch], ct, arch); if (SQLite_print_contents_subtree(db, ct->filetree[arch], ct, arch)) return 1;
SQLite_commit_transaction(db);
/* if (sqlite3_finalize(stmt)) { /* if (sqlite3_finalize(stmt)) {
fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db)); fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db));
}*/ }*/
@ -125,21 +124,29 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) {
while (currpackage) { while (currpackage) {
/* packages <-> files relations */ /* packages <-> files relations */
for (i = 0; i < currpackage->filenamecount; i++) { for (i = 0; i < currpackage->filenamecount; i++) {
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%d,%d);", snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%d,%d,?,?,?);",
currpackage->id, currpackage->id,
currpackage->file[arch]->id); currpackage->file[i]->id);
if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) { if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) {
fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db));
} }
sqlite3_bind_text(stmt, 1, currpackage->name, -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, currpackage->version, -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 3, currpackage->release, -1, SQLITE_STATIC);
if (sqlite3_step(stmt) != SQLITE_DONE) { if (sqlite3_step(stmt) != SQLITE_DONE) {
fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db));
return 3; return 3;
} }
sqlite3_finalize(stmt); sqlite3_finalize(stmt);
sqlite3_transaction_size += strlen(sqlite3_query);
if (sqlite3_transaction_size >= sqlite3_transaction_maxsize) {
SQLite_commit_transaction(db);
SQLite_begin_transaction(db);
}
} }
currpackage = currpackage->next; currpackage = currpackage->next;
} }
SQLite_commit_transaction(db);
return 0; return 0;
} }
@ -503,6 +510,28 @@ generateSQLite_sources(struct configTag *ct, sqlite3 *db) {
SQLite_commit_transaction(db); SQLite_commit_transaction(db);
} }
int
generateSQLiteFiles(struct configTag *ct)
{
sqlite3 *db = NULL;
char dbname[PATH_MAX];
int i;
for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) {
snprintf(dbname, PATH_MAX, "%s%s-%s-files.db", ct->repository_dir, ct->tag, ct->arch[i]);
unlink(dbname);
if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) {
logmsg(LOG_ERROR, "sqlite3_open_v2: %s %s", sqlite3_errmsg(db), dbname);
if (db) sqlite3_close(db);
return 1;
}
generateSQLite_files(ct, db, i);
sqlite3_close(db);
}
return 0;
}
int int
generateSQLite(struct configTag *ct) generateSQLite(struct configTag *ct)
{ {
@ -521,7 +550,6 @@ generateSQLite(struct configTag *ct)
} }
generateSQLite_packages(ct, db, i); generateSQLite_packages(ct, db, i);
// generateSQLite_files(ct, db);
generateSQLite_provided(ct, db, i); generateSQLite_provided(ct, db, i);
sqlite3_close(db); sqlite3_close(db);
} }

27
src/distromatic.c
View File

@ -86,6 +86,7 @@
#define MODE_GENBUILDINFO 32 #define MODE_GENBUILDINFO 32
#define MODE_GENPKGLIST 64 #define MODE_GENPKGLIST 64
#define MODE_SQLITE3 128 #define MODE_SQLITE3 128
#define MODE_SQLITE3_FILES 256
static void program_usage(int exit_code); static void program_usage(int exit_code);
static void program_version(void); static void program_version(void);
@ -139,7 +140,8 @@ static const char *helpmsg[] = {
" --genhtml generate HTML code for repository", " --genhtml generate HTML code for repository",
" --genpkglist generate binary packages list with version and size", " --genpkglist generate binary packages list with version and size",
" --gensrcpkglist generate a source packages list with version", " --gensrcpkglist generate a source packages list with version",
" --gensqlite3 dump data to an SQLite3 database (EXPERIMENTAL)", " --gensqlite3 dump data to SQLite3 databases",
" --gensqlite3files dump files to SQLite3 databases",
" --find-deps <package> find dependencies for given package name", " --find-deps <package> find dependencies for given package name",
" --changelog <package> print changelog for given package name", " --changelog <package> print changelog for given package name",
" --changelogsince <mmddyy> print changelog for all packages since given date", " --changelogsince <mmddyy> print changelog for all packages since given date",
@ -948,6 +950,7 @@ main(int argc, char *argv[])
{ "genpkglist", no_argument, 0, 0 }, { "genpkglist", no_argument, 0, 0 },
{ "gensrcpkglist", no_argument, 0, 0 }, { "gensrcpkglist", no_argument, 0, 0 },
{ "gensqlite3", no_argument, 0, 0 }, { "gensqlite3", no_argument, 0, 0 },
{ "gensqlite3files", no_argument, 0, 0 },
{ "arch", required_argument, 0, 'a' }, { "arch", required_argument, 0, 'a' },
{ "conf", required_argument, 0, 'c' }, { "conf", required_argument, 0, 'c' },
{ "help", no_argument, 0, 'h' }, { "help", no_argument, 0, 'h' },
@ -1000,6 +1003,8 @@ main(int argc, char *argv[])
name = NULL; name = NULL;
} else if (!strcmp(longopts[longindex].name, "gensqlite3")) { } else if (!strcmp(longopts[longindex].name, "gensqlite3")) {
mode |= MODE_SQLITE3; mode |= MODE_SQLITE3;
} else if (!strcmp(longopts[longindex].name, "gensqlite3files")) {
mode |= MODE_SQLITE3_FILES;
} else if (!strcmp(longopts[longindex].name, "gensrcpkglist")) { } else if (!strcmp(longopts[longindex].name, "gensrcpkglist")) {
mode |= MODE_GENSRCPKGLIST; mode |= MODE_GENSRCPKGLIST;
name = NULL; name = NULL;
@ -1058,7 +1063,7 @@ main(int argc, char *argv[])
} }
/* if only target is sqlite3 db minimize operations */ /* if only target is sqlite3 db minimize operations */
if (mode == MODE_SQLITE3) { if (mode == MODE_SQLITE3 || mode == MODE_SQLITE3_FILES || mode == (MODE_SQLITE3 | MODE_SQLITE3_FILES)) {
genheader_mode |= GENHEADER_CHANGELOG; genheader_mode |= GENHEADER_CHANGELOG;
recursive_mode = 0; recursive_mode = 0;
incremental_mode = 0; incremental_mode = 0;
@ -1198,13 +1203,23 @@ main(int argc, char *argv[])
} // if (genheader_mode) } // if (genheader_mode)
if (mode & MODE_SQLITE3) { if (mode & MODE_SQLITE3) {
if (!quietmode) printf("Generating sqlite databases...\n"); if (!quietmode) printf("Generating sqlite3 databases...\n");
logmsg(LOG_DEBUG,"generateSQLite - start"); logmsg(LOG_DEBUG,"generateSQLite3 - start");
if (generateSQLite(configtag)) { if (generateSQLite(configtag)) {
logmsg(LOG_ERROR, "could not generate sqlite databases; aborting."); logmsg(LOG_ERROR, "could not generate sqlite3 databases; aborting.");
exit(1); exit(1);
} }
logmsg(LOG_DEBUG,"generateSQLite - done"); logmsg(LOG_DEBUG,"generateSQLite3 - done");
}
if (mode & MODE_SQLITE3_FILES) {
if (!quietmode) printf("Generating sqlite3 files databases...\n");
logmsg(LOG_DEBUG,"generateSQLite3_files - start");
if (generateSQLiteFiles(configtag)) {
logmsg(LOG_ERROR, "could not generate sqlite3 files databases; aborting.");
exit(1);
}
logmsg(LOG_DEBUG,"generateSQLite3_files - done");
} }
if (genheader_mode & GENHEADER_STATS) { if (genheader_mode & GENHEADER_STATS) {