diff --git a/src/backend-sqlite3.c b/src/backend-sqlite3.c index 1c64fb2..35b87e3 100644 --- a/src/backend-sqlite3.c +++ b/src/backend-sqlite3.c @@ -26,6 +26,8 @@ char sqlite3_query[PATH_MAX]; sqlite3_stmt *stmt; + +#define sqlite3_transaction_maxsize 10485760 long sqlite3_transaction_size; int SQLite_init_table(sqlite3 *db, const char* table_name, const char* table_query) { @@ -61,46 +63,44 @@ int SQLite_commit_transaction(sqlite3 *db) { return 0; } -void SQLite_print_contents_subtree(sqlite3 *db, +int SQLite_print_contents_subtree(sqlite3 *db, struct fileTree* ft, struct configTag* ct, int arch) { while (ft) { - snprintf(sqlite3_query, PATH_MAX, "INSERT INTO files_%s VALUES(%d,?,%d,%d,%d,%d);", - ct->arch[arch], + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO files VALUES(%d,?,%d,%d,%d,%d);", ft->id, (ft->firstchild?ft->firstchild->id:-1), (ft->next?ft->next->id:-1), (ft->parent?ft->parent->id:-1), ft->numproviders); - sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL); sqlite3_bind_text(stmt, 1, ft->name, -1, SQLITE_STATIC); if (sqlite3_step(stmt) != SQLITE_DONE) { - fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + fprintf(stderr, "ERROR: SQLite: %s (%s)\n", sqlite3_errmsg(db), sqlite3_query); } -/* sqlite3_transaction_size += strlen(sqlite3_query); - if (sqlite3_transaction_size >= 655360) { - fprintf(stderr, "Commit: %ld\n", sqlite3_transaction_size); - sqlite3_exec(db, "COMMIT;", 0, 0, 0); - sqlite3_transaction_size = 0; - }*/ + sqlite3_transaction_size += strlen(sqlite3_query); + if (sqlite3_transaction_size >= sqlite3_transaction_maxsize) { + SQLite_commit_transaction(db); + SQLite_begin_transaction(db); + } if (ft->firstchild) { - SQLite_print_contents_subtree(db, ft->firstchild, ct, arch); + if (SQLite_print_contents_subtree(db, ft->firstchild, ct, arch)) return 1; } ft=ft->next; } + return 0; } #define SQLITE_TABLE_files "id INTEGER PRIMARY KEY, "\ "name STRING, firstchild INTEGER, next INTEGER, parent INTEGER, numproviders INTEGER" /* struct headerList **provider; */ -#define SQLITE_TABLE_packages_files_rel "id INTEGER PRIMARY KEY, id_package INTEGER, id_file INTEGER" +#define SQLITE_TABLE_packages_files_rel "id INTEGER PRIMARY KEY, id_package INTEGER, id_file INTEGER, name STRING, version STRING, release STRING" int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) { @@ -114,9 +114,8 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) { SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_files); SQLite_begin_transaction(db); - SQLite_print_contents_subtree(db, ct->filetree[arch], ct, arch); - SQLite_commit_transaction(db); - + if (SQLite_print_contents_subtree(db, ct->filetree[arch], ct, arch)) return 1; + /* if (sqlite3_finalize(stmt)) { fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db)); }*/ @@ -125,21 +124,29 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) { while (currpackage) { /* packages <-> files relations */ for (i = 0; i < currpackage->filenamecount; i++) { - snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%d,%d);", + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%d,%d,?,?,?);", currpackage->id, - currpackage->file[arch]->id); - + currpackage->file[i]->id); if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) { fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); } + sqlite3_bind_text(stmt, 1, currpackage->name, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, currpackage->version, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, currpackage->release, -1, SQLITE_STATIC); if (sqlite3_step(stmt) != SQLITE_DONE) { fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); return 3; } sqlite3_finalize(stmt); + sqlite3_transaction_size += strlen(sqlite3_query); + if (sqlite3_transaction_size >= sqlite3_transaction_maxsize) { + SQLite_commit_transaction(db); + SQLite_begin_transaction(db); + } } currpackage = currpackage->next; } + SQLite_commit_transaction(db); return 0; } @@ -503,6 +510,28 @@ generateSQLite_sources(struct configTag *ct, sqlite3 *db) { SQLite_commit_transaction(db); } +int +generateSQLiteFiles(struct configTag *ct) +{ + sqlite3 *db = NULL; + char dbname[PATH_MAX]; + int i; + + for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) { + snprintf(dbname, PATH_MAX, "%s%s-%s-files.db", ct->repository_dir, ct->tag, ct->arch[i]); + unlink(dbname); + + if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) { + logmsg(LOG_ERROR, "sqlite3_open_v2: %s %s", sqlite3_errmsg(db), dbname); + if (db) sqlite3_close(db); + return 1; + } + generateSQLite_files(ct, db, i); + sqlite3_close(db); + } + return 0; +} + int generateSQLite(struct configTag *ct) { @@ -521,7 +550,6 @@ generateSQLite(struct configTag *ct) } generateSQLite_packages(ct, db, i); -// generateSQLite_files(ct, db); generateSQLite_provided(ct, db, i); sqlite3_close(db); } diff --git a/src/distromatic.c b/src/distromatic.c index 9b0f7be..ebef2aa 100644 --- a/src/distromatic.c +++ b/src/distromatic.c @@ -86,6 +86,7 @@ #define MODE_GENBUILDINFO 32 #define MODE_GENPKGLIST 64 #define MODE_SQLITE3 128 +#define MODE_SQLITE3_FILES 256 static void program_usage(int exit_code); static void program_version(void); @@ -139,7 +140,8 @@ static const char *helpmsg[] = { " --genhtml generate HTML code for repository", " --genpkglist generate binary packages list with version and size", " --gensrcpkglist generate a source packages list with version", -" --gensqlite3 dump data to an SQLite3 database (EXPERIMENTAL)", +" --gensqlite3 dump data to SQLite3 databases", +" --gensqlite3files dump files to SQLite3 databases", " --find-deps find dependencies for given package name", " --changelog print changelog for given package name", " --changelogsince print changelog for all packages since given date", @@ -948,6 +950,7 @@ main(int argc, char *argv[]) { "genpkglist", no_argument, 0, 0 }, { "gensrcpkglist", no_argument, 0, 0 }, { "gensqlite3", no_argument, 0, 0 }, + { "gensqlite3files", no_argument, 0, 0 }, { "arch", required_argument, 0, 'a' }, { "conf", required_argument, 0, 'c' }, { "help", no_argument, 0, 'h' }, @@ -1000,6 +1003,8 @@ main(int argc, char *argv[]) name = NULL; } else if (!strcmp(longopts[longindex].name, "gensqlite3")) { mode |= MODE_SQLITE3; + } else if (!strcmp(longopts[longindex].name, "gensqlite3files")) { + mode |= MODE_SQLITE3_FILES; } else if (!strcmp(longopts[longindex].name, "gensrcpkglist")) { mode |= MODE_GENSRCPKGLIST; name = NULL; @@ -1058,7 +1063,7 @@ main(int argc, char *argv[]) } /* if only target is sqlite3 db minimize operations */ - if (mode == MODE_SQLITE3) { + if (mode == MODE_SQLITE3 || mode == MODE_SQLITE3_FILES || mode == (MODE_SQLITE3 | MODE_SQLITE3_FILES)) { genheader_mode |= GENHEADER_CHANGELOG; recursive_mode = 0; incremental_mode = 0; @@ -1198,13 +1203,23 @@ main(int argc, char *argv[]) } // if (genheader_mode) if (mode & MODE_SQLITE3) { - if (!quietmode) printf("Generating sqlite databases...\n"); - logmsg(LOG_DEBUG,"generateSQLite - start"); + if (!quietmode) printf("Generating sqlite3 databases...\n"); + logmsg(LOG_DEBUG,"generateSQLite3 - start"); if (generateSQLite(configtag)) { - logmsg(LOG_ERROR, "could not generate sqlite databases; aborting."); + logmsg(LOG_ERROR, "could not generate sqlite3 databases; aborting."); exit(1); } - logmsg(LOG_DEBUG,"generateSQLite - done"); + logmsg(LOG_DEBUG,"generateSQLite3 - done"); + } + + if (mode & MODE_SQLITE3_FILES) { + if (!quietmode) printf("Generating sqlite3 files databases...\n"); + logmsg(LOG_DEBUG,"generateSQLite3_files - start"); + if (generateSQLiteFiles(configtag)) { + logmsg(LOG_ERROR, "could not generate sqlite3 files databases; aborting."); + exit(1); + } + logmsg(LOG_DEBUG,"generateSQLite3_files - done"); } if (genheader_mode & GENHEADER_STATS) {