backend-sqlite3: add --gensqlite3files option to generate <repository>-<arch>-files.db databases
This commit is contained in:
parent
2b99d7d46f
commit
ec0c093578
@ -26,6 +26,8 @@
|
|||||||
|
|
||||||
char sqlite3_query[PATH_MAX];
|
char sqlite3_query[PATH_MAX];
|
||||||
sqlite3_stmt *stmt;
|
sqlite3_stmt *stmt;
|
||||||
|
|
||||||
|
#define sqlite3_transaction_maxsize 10485760
|
||||||
long sqlite3_transaction_size;
|
long sqlite3_transaction_size;
|
||||||
|
|
||||||
int SQLite_init_table(sqlite3 *db, const char* table_name, const char* table_query) {
|
int SQLite_init_table(sqlite3 *db, const char* table_name, const char* table_query) {
|
||||||
@ -61,46 +63,44 @@ int SQLite_commit_transaction(sqlite3 *db) {
|
|||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
void SQLite_print_contents_subtree(sqlite3 *db,
|
int SQLite_print_contents_subtree(sqlite3 *db,
|
||||||
struct fileTree* ft,
|
struct fileTree* ft,
|
||||||
struct configTag* ct,
|
struct configTag* ct,
|
||||||
int arch) {
|
int arch) {
|
||||||
|
|
||||||
while (ft) {
|
while (ft) {
|
||||||
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO files_%s VALUES(%d,?,%d,%d,%d,%d);",
|
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO files VALUES(%d,?,%d,%d,%d,%d);",
|
||||||
ct->arch[arch],
|
|
||||||
ft->id,
|
ft->id,
|
||||||
(ft->firstchild?ft->firstchild->id:-1),
|
(ft->firstchild?ft->firstchild->id:-1),
|
||||||
(ft->next?ft->next->id:-1),
|
(ft->next?ft->next->id:-1),
|
||||||
(ft->parent?ft->parent->id:-1),
|
(ft->parent?ft->parent->id:-1),
|
||||||
ft->numproviders);
|
ft->numproviders);
|
||||||
|
|
||||||
sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL);
|
sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL);
|
||||||
sqlite3_bind_text(stmt, 1, ft->name, -1, SQLITE_STATIC);
|
sqlite3_bind_text(stmt, 1, ft->name, -1, SQLITE_STATIC);
|
||||||
|
|
||||||
if (sqlite3_step(stmt) != SQLITE_DONE) {
|
if (sqlite3_step(stmt) != SQLITE_DONE) {
|
||||||
fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db));
|
fprintf(stderr, "ERROR: SQLite: %s (%s)\n", sqlite3_errmsg(db), sqlite3_query);
|
||||||
}
|
}
|
||||||
|
|
||||||
/* sqlite3_transaction_size += strlen(sqlite3_query);
|
sqlite3_transaction_size += strlen(sqlite3_query);
|
||||||
if (sqlite3_transaction_size >= 655360) {
|
if (sqlite3_transaction_size >= sqlite3_transaction_maxsize) {
|
||||||
fprintf(stderr, "Commit: %ld\n", sqlite3_transaction_size);
|
SQLite_commit_transaction(db);
|
||||||
sqlite3_exec(db, "COMMIT;", 0, 0, 0);
|
SQLite_begin_transaction(db);
|
||||||
sqlite3_transaction_size = 0;
|
}
|
||||||
}*/
|
|
||||||
|
|
||||||
if (ft->firstchild) {
|
if (ft->firstchild) {
|
||||||
SQLite_print_contents_subtree(db, ft->firstchild, ct, arch);
|
if (SQLite_print_contents_subtree(db, ft->firstchild, ct, arch)) return 1;
|
||||||
}
|
}
|
||||||
ft=ft->next;
|
ft=ft->next;
|
||||||
}
|
}
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
#define SQLITE_TABLE_files "id INTEGER PRIMARY KEY, "\
|
#define SQLITE_TABLE_files "id INTEGER PRIMARY KEY, "\
|
||||||
"name STRING, firstchild INTEGER, next INTEGER, parent INTEGER, numproviders INTEGER"
|
"name STRING, firstchild INTEGER, next INTEGER, parent INTEGER, numproviders INTEGER"
|
||||||
/* struct headerList **provider; */
|
/* struct headerList **provider; */
|
||||||
|
|
||||||
#define SQLITE_TABLE_packages_files_rel "id INTEGER PRIMARY KEY, id_package INTEGER, id_file INTEGER"
|
#define SQLITE_TABLE_packages_files_rel "id INTEGER PRIMARY KEY, id_package INTEGER, id_file INTEGER, name STRING, version STRING, release STRING"
|
||||||
|
|
||||||
int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) {
|
int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) {
|
||||||
|
|
||||||
@ -114,8 +114,7 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) {
|
|||||||
SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_files);
|
SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_files);
|
||||||
|
|
||||||
SQLite_begin_transaction(db);
|
SQLite_begin_transaction(db);
|
||||||
SQLite_print_contents_subtree(db, ct->filetree[arch], ct, arch);
|
if (SQLite_print_contents_subtree(db, ct->filetree[arch], ct, arch)) return 1;
|
||||||
SQLite_commit_transaction(db);
|
|
||||||
|
|
||||||
/* if (sqlite3_finalize(stmt)) {
|
/* if (sqlite3_finalize(stmt)) {
|
||||||
fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db));
|
fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db));
|
||||||
@ -125,21 +124,29 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) {
|
|||||||
while (currpackage) {
|
while (currpackage) {
|
||||||
/* packages <-> files relations */
|
/* packages <-> files relations */
|
||||||
for (i = 0; i < currpackage->filenamecount; i++) {
|
for (i = 0; i < currpackage->filenamecount; i++) {
|
||||||
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%d,%d);",
|
snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%d,%d,?,?,?);",
|
||||||
currpackage->id,
|
currpackage->id,
|
||||||
currpackage->file[arch]->id);
|
currpackage->file[i]->id);
|
||||||
|
|
||||||
if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) {
|
if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) {
|
||||||
fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db));
|
fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db));
|
||||||
}
|
}
|
||||||
|
sqlite3_bind_text(stmt, 1, currpackage->name, -1, SQLITE_STATIC);
|
||||||
|
sqlite3_bind_text(stmt, 2, currpackage->version, -1, SQLITE_STATIC);
|
||||||
|
sqlite3_bind_text(stmt, 3, currpackage->release, -1, SQLITE_STATIC);
|
||||||
if (sqlite3_step(stmt) != SQLITE_DONE) {
|
if (sqlite3_step(stmt) != SQLITE_DONE) {
|
||||||
fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db));
|
fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db));
|
||||||
return 3;
|
return 3;
|
||||||
}
|
}
|
||||||
sqlite3_finalize(stmt);
|
sqlite3_finalize(stmt);
|
||||||
|
sqlite3_transaction_size += strlen(sqlite3_query);
|
||||||
|
if (sqlite3_transaction_size >= sqlite3_transaction_maxsize) {
|
||||||
|
SQLite_commit_transaction(db);
|
||||||
|
SQLite_begin_transaction(db);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
currpackage = currpackage->next;
|
currpackage = currpackage->next;
|
||||||
}
|
}
|
||||||
|
SQLite_commit_transaction(db);
|
||||||
|
|
||||||
return 0;
|
return 0;
|
||||||
}
|
}
|
||||||
@ -503,6 +510,28 @@ generateSQLite_sources(struct configTag *ct, sqlite3 *db) {
|
|||||||
SQLite_commit_transaction(db);
|
SQLite_commit_transaction(db);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
int
|
||||||
|
generateSQLiteFiles(struct configTag *ct)
|
||||||
|
{
|
||||||
|
sqlite3 *db = NULL;
|
||||||
|
char dbname[PATH_MAX];
|
||||||
|
int i;
|
||||||
|
|
||||||
|
for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) {
|
||||||
|
snprintf(dbname, PATH_MAX, "%s%s-%s-files.db", ct->repository_dir, ct->tag, ct->arch[i]);
|
||||||
|
unlink(dbname);
|
||||||
|
|
||||||
|
if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) {
|
||||||
|
logmsg(LOG_ERROR, "sqlite3_open_v2: %s %s", sqlite3_errmsg(db), dbname);
|
||||||
|
if (db) sqlite3_close(db);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
generateSQLite_files(ct, db, i);
|
||||||
|
sqlite3_close(db);
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
int
|
int
|
||||||
generateSQLite(struct configTag *ct)
|
generateSQLite(struct configTag *ct)
|
||||||
{
|
{
|
||||||
@ -521,7 +550,6 @@ generateSQLite(struct configTag *ct)
|
|||||||
}
|
}
|
||||||
|
|
||||||
generateSQLite_packages(ct, db, i);
|
generateSQLite_packages(ct, db, i);
|
||||||
// generateSQLite_files(ct, db);
|
|
||||||
generateSQLite_provided(ct, db, i);
|
generateSQLite_provided(ct, db, i);
|
||||||
sqlite3_close(db);
|
sqlite3_close(db);
|
||||||
}
|
}
|
||||||
|
@ -86,6 +86,7 @@
|
|||||||
#define MODE_GENBUILDINFO 32
|
#define MODE_GENBUILDINFO 32
|
||||||
#define MODE_GENPKGLIST 64
|
#define MODE_GENPKGLIST 64
|
||||||
#define MODE_SQLITE3 128
|
#define MODE_SQLITE3 128
|
||||||
|
#define MODE_SQLITE3_FILES 256
|
||||||
|
|
||||||
static void program_usage(int exit_code);
|
static void program_usage(int exit_code);
|
||||||
static void program_version(void);
|
static void program_version(void);
|
||||||
@ -139,7 +140,8 @@ static const char *helpmsg[] = {
|
|||||||
" --genhtml generate HTML code for repository",
|
" --genhtml generate HTML code for repository",
|
||||||
" --genpkglist generate binary packages list with version and size",
|
" --genpkglist generate binary packages list with version and size",
|
||||||
" --gensrcpkglist generate a source packages list with version",
|
" --gensrcpkglist generate a source packages list with version",
|
||||||
" --gensqlite3 dump data to an SQLite3 database (EXPERIMENTAL)",
|
" --gensqlite3 dump data to SQLite3 databases",
|
||||||
|
" --gensqlite3files dump files to SQLite3 databases",
|
||||||
" --find-deps <package> find dependencies for given package name",
|
" --find-deps <package> find dependencies for given package name",
|
||||||
" --changelog <package> print changelog for given package name",
|
" --changelog <package> print changelog for given package name",
|
||||||
" --changelogsince <mmddyy> print changelog for all packages since given date",
|
" --changelogsince <mmddyy> print changelog for all packages since given date",
|
||||||
@ -948,6 +950,7 @@ main(int argc, char *argv[])
|
|||||||
{ "genpkglist", no_argument, 0, 0 },
|
{ "genpkglist", no_argument, 0, 0 },
|
||||||
{ "gensrcpkglist", no_argument, 0, 0 },
|
{ "gensrcpkglist", no_argument, 0, 0 },
|
||||||
{ "gensqlite3", no_argument, 0, 0 },
|
{ "gensqlite3", no_argument, 0, 0 },
|
||||||
|
{ "gensqlite3files", no_argument, 0, 0 },
|
||||||
{ "arch", required_argument, 0, 'a' },
|
{ "arch", required_argument, 0, 'a' },
|
||||||
{ "conf", required_argument, 0, 'c' },
|
{ "conf", required_argument, 0, 'c' },
|
||||||
{ "help", no_argument, 0, 'h' },
|
{ "help", no_argument, 0, 'h' },
|
||||||
@ -1000,6 +1003,8 @@ main(int argc, char *argv[])
|
|||||||
name = NULL;
|
name = NULL;
|
||||||
} else if (!strcmp(longopts[longindex].name, "gensqlite3")) {
|
} else if (!strcmp(longopts[longindex].name, "gensqlite3")) {
|
||||||
mode |= MODE_SQLITE3;
|
mode |= MODE_SQLITE3;
|
||||||
|
} else if (!strcmp(longopts[longindex].name, "gensqlite3files")) {
|
||||||
|
mode |= MODE_SQLITE3_FILES;
|
||||||
} else if (!strcmp(longopts[longindex].name, "gensrcpkglist")) {
|
} else if (!strcmp(longopts[longindex].name, "gensrcpkglist")) {
|
||||||
mode |= MODE_GENSRCPKGLIST;
|
mode |= MODE_GENSRCPKGLIST;
|
||||||
name = NULL;
|
name = NULL;
|
||||||
@ -1058,7 +1063,7 @@ main(int argc, char *argv[])
|
|||||||
}
|
}
|
||||||
|
|
||||||
/* if only target is sqlite3 db minimize operations */
|
/* if only target is sqlite3 db minimize operations */
|
||||||
if (mode == MODE_SQLITE3) {
|
if (mode == MODE_SQLITE3 || mode == MODE_SQLITE3_FILES || mode == (MODE_SQLITE3 | MODE_SQLITE3_FILES)) {
|
||||||
genheader_mode |= GENHEADER_CHANGELOG;
|
genheader_mode |= GENHEADER_CHANGELOG;
|
||||||
recursive_mode = 0;
|
recursive_mode = 0;
|
||||||
incremental_mode = 0;
|
incremental_mode = 0;
|
||||||
@ -1198,13 +1203,23 @@ main(int argc, char *argv[])
|
|||||||
} // if (genheader_mode)
|
} // if (genheader_mode)
|
||||||
|
|
||||||
if (mode & MODE_SQLITE3) {
|
if (mode & MODE_SQLITE3) {
|
||||||
if (!quietmode) printf("Generating sqlite databases...\n");
|
if (!quietmode) printf("Generating sqlite3 databases...\n");
|
||||||
logmsg(LOG_DEBUG,"generateSQLite - start");
|
logmsg(LOG_DEBUG,"generateSQLite3 - start");
|
||||||
if (generateSQLite(configtag)) {
|
if (generateSQLite(configtag)) {
|
||||||
logmsg(LOG_ERROR, "could not generate sqlite databases; aborting.");
|
logmsg(LOG_ERROR, "could not generate sqlite3 databases; aborting.");
|
||||||
exit(1);
|
exit(1);
|
||||||
}
|
}
|
||||||
logmsg(LOG_DEBUG,"generateSQLite - done");
|
logmsg(LOG_DEBUG,"generateSQLite3 - done");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (mode & MODE_SQLITE3_FILES) {
|
||||||
|
if (!quietmode) printf("Generating sqlite3 files databases...\n");
|
||||||
|
logmsg(LOG_DEBUG,"generateSQLite3_files - start");
|
||||||
|
if (generateSQLiteFiles(configtag)) {
|
||||||
|
logmsg(LOG_ERROR, "could not generate sqlite3 files databases; aborting.");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
logmsg(LOG_DEBUG,"generateSQLite3_files - done");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (genheader_mode & GENHEADER_STATS) {
|
if (genheader_mode & GENHEADER_STATS) {
|
||||||
|
Loading…
Reference in New Issue
Block a user