From 944c8673eaf8177bfe812f8a35281b68ceaae5cc Mon Sep 17 00:00:00 2001 From: Silvan Calarco Date: Mon, 4 Mar 2013 15:08:13 +0100 Subject: [PATCH] Added new sqlite3 backend (experimental and to be completed) --- configure.ac | 3 + src/Makefile.am | 5 +- src/backend-sqlite3.c | 363 ++++++++++++++++++++++++++++++++++ src/distromatic.c | 17 ++ src/headerlist.c | 5 + src/include/backend-sqlite3.h | 14 ++ src/include/changelog.h | 1 + src/include/headerlist.h | 3 + 8 files changed, 410 insertions(+), 1 deletion(-) create mode 100644 src/backend-sqlite3.c create mode 100644 src/include/backend-sqlite3.h diff --git a/configure.ac b/configure.ac index d1abb85..a8215a4 100644 --- a/configure.ac +++ b/configure.ac @@ -218,6 +218,9 @@ AC_CHECK_LIB(iberty, freeargv, [AC_MSG_ERROR([Can't find libiberty in your system,] [check http://www.gnu.org/software/binutils/binutils.html for libiberty])]) +dnl Check for sqlite library and header files +PKG_CHECK_MODULES([SQLITE], [sqlite3]) + dnl Set the default `prefix' variable to `/usr' dnl ------------------------------------------------------------ #AC_PREFIX_DEFAULT([/usr]) diff --git a/src/Makefile.am b/src/Makefile.am index a1fc896..3af4073 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -17,6 +17,8 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/include -I$(top_builddir)/src/include +LDFLAGS += $(SQLITE_LIBS) + bin_PROGRAMS = distromatic distromatic_SOURCES = \ @@ -27,6 +29,7 @@ distromatic_SOURCES = \ reports.c \ headerlist.c \ requirelist.c \ - rpmfunctions.c + rpmfunctions.c \ + backend-sqlite3.c SUBDIRS = include missing diff --git a/src/backend-sqlite3.c b/src/backend-sqlite3.c new file mode 100644 index 0000000..4042630 --- /dev/null +++ b/src/backend-sqlite3.c @@ -0,0 +1,363 @@ +/* + * distromatic - tool for RPM based repositories + * + * Copyright (C) 2013 by Silvan Calarco + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of version 2 of the GNU General Public License as published by the + * Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY, to the extent permitted by law; without even the implied + * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + * See the GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. + */ + +#include +#include "config.h" +#include "headerlist.h" +#include "changelog.h" +#include "backend-sqlite3.h" + +#define SQLITE_TABLE_files "id INTEGER PRIMARY KEY, "\ + "name STRING, firstchild INTEGER, next INTEGER, parent INTEGER, numproviders INTEGER" +/* struct headerList **provider; +};*/ + +#define SQLITE_TABLE_provided "id INTEGER PRIMARY KEY, "\ + "name STRING, flags INTEGER, numproviders INTEGER" +/*struct providedList { + struct headerList **provider; + int numbuildproviders; + struct headerList **buildprovider; + int numversions; + char **version; + int buildpriority; + struct providedList *next; +};*/ + +#define SQLITE_TABLE_packager "id INTEGER PRIMARY KEY, "\ + "name STRING, role INTEGER, changes_count INTEGER, packages_count INTEGER" +/*struct Packager { + char *alias[PACKAGER_MAXALIASES]; +};*/ + +#define SQLITE_TABLE_sources_source "id INTEGER PRIMARY KEY, id_sources INTEGER, source STRING" +#define SQLITE_TABLE_sources_patch "id INTEGER PRIMARY KEY, id_sources INTEGER, patch STRING" +//#define SQLITE_TABLE_sources_files_rel "id INTEGER PRIMARY KEY, id_sources INTEGER, id_files INTEGER" +#define SQLITE_TABLE_sources_provided_rel "id INTEGER PRIMARY KEY, id_sources INTEGER, id_provided INTEGER" + +#define SQLITE_TABLE_sources "id INTEGER PRIMARY KEY, "\ + "name STRING, altrepository INTEGER, epoch INTEGER, version STRING, release STRING, summary STRING, id_packager INTEGER, "\ + "groupdescr STRING, description STRING, url STRING, license STRING, arch STRING, buildarchs STRING, " \ + "excludearch STRING, buildtime INTEGER" +/* long size; + char **dirname; + char **basename; + int filenamecount; + int requirecount; + struct Require **require; + struct providedList **required; + int providecount; + struct requireList *requirelist; + struct headerSourceList *old; + struct changeLog *changelog; + struct headerList *firstchild[ARCHS_MAX]; + struct warningList *firstwarning; + struct headerSourceList *updatingparent; +};*/ + + +char sqlite3_query[PATH_MAX]; +sqlite3_stmt *stmt; +long sqlite3_transaction_size; + +int SQLite_init_table(sqlite3 *db, const char* table_name, const char* table_query) { + + char myquery[PATH_MAX]; + + snprintf(myquery, PATH_MAX, "CREATE TABLE IF NOT EXISTS %s(%s)", table_name, table_query); + + if (sqlite3_exec(db, myquery, 0, 0, 0)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", myquery, sqlite3_errmsg(db)); + return 2; + } + return 0; +} + +int SQLite_begin_transaction(sqlite3 *db) { + + if (sqlite3_exec(db, "BEGIN TRANSACTION;", 0, 0, 0)) { + fprintf(stderr, "ERROR: SQLite: (BEGIN TRANSACTION) %s\n", sqlite3_errmsg(db)); + return 1; + } + + sqlite3_transaction_size = 0; + return 0; +} + +int SQLite_commit_transaction(sqlite3 *db) { + if (sqlite3_exec(db, "COMMIT;", 0, 0, 0)) { + fprintf(stderr, "ERROR: SQLite: (COMMIT) %s\n", sqlite3_errmsg(db)); + return 1; + } + sqlite3_transaction_size = 0; + return 0; +} + +void SQLite_print_contents_subtree(sqlite3 *db, + struct fileTree* ft, + struct configTag* ct, + int arch) { + + while (ft) { + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO files_%s VALUES(%d,?,%d,%d,%d,%d);", + ct->arch[arch], + ft->id, + (ft->firstchild?ft->firstchild->id:-1), + (ft->next?ft->next->id:-1), + (ft->parent?ft->parent->id:-1), + ft->numproviders); + + sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL); + sqlite3_bind_text(stmt, 1, ft->name, -1, SQLITE_STATIC); + + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + } + +/* sqlite3_transaction_size += strlen(sqlite3_query); + if (sqlite3_transaction_size >= 655360) { + fprintf(stderr, "Commit: %ld\n", sqlite3_transaction_size); + sqlite3_exec(db, "COMMIT;", 0, 0, 0); + sqlite3_transaction_size = 0; + }*/ + + if (ft->firstchild) { + SQLite_print_contents_subtree(db, ft->firstchild, ct, arch); + } + ft=ft->next; + } +} + +int generateSQLite_files(struct configTag* ct, sqlite3 *db) { + + int i; + + for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) { + snprintf(sqlite3_query, PATH_MAX, "files_%s", ct->arch[i]); + SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_files); + SQLite_begin_transaction(db); + SQLite_print_contents_subtree(db, ct->filetree[i], ct, i); + SQLite_commit_transaction(db); + +/* if (sqlite3_finalize(stmt)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db)); + }*/ + } + + return 0; +} + + +int generateSQLite_provided(struct configTag* ct, sqlite3 *db) { + + int i; + struct providedList* provided; + + for (i = 0; i < ARCHS_MAX && ct->arch[i]; i++) { + snprintf(sqlite3_query, PATH_MAX, "provided_%s", ct->arch[i]); + SQLite_init_table(db, sqlite3_query, SQLITE_TABLE_provided); + SQLite_begin_transaction(db); + + provided = ct->providedlist_idx[i][0]; + while (provided) { + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO provided_%s VALUES(%d,?,%d,%d);", + ct->arch[i], + provided->id, + provided->flags, + provided->numproviders); + + sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL); + sqlite3_bind_text(stmt, 1, provided->name, -1, SQLITE_STATIC); + + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + } + provided = provided->next; + } + + SQLite_commit_transaction(db); + +/* if (sqlite3_finalize(stmt)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db)); + }*/ + } + + return 0; +} + +int generateSQLite_packager(sqlite3 *db) { + + struct Packager* packager = firstPackager(); + + SQLite_init_table(db, "packager", SQLITE_TABLE_packager); + SQLite_begin_transaction(db); + + while (packager) { + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packager VALUES(NULL,?,%d,%d,%d);", + packager->role, + packager->changes_count, + packager->packages_count); + + if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + } + sqlite3_bind_text(stmt, 1, packager->name, -1, SQLITE_STATIC); + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + return 3; + } + packager->id = sqlite3_last_insert_rowid(db); + sqlite3_finalize(stmt); + packager = packager->next; + } + SQLite_commit_transaction(db); +} + +int +generateSQLite_sources(struct configTag *ct, sqlite3 *db) { + + struct headerSourceList* currsource; + int i; + + SQLite_init_table(db, "sources", SQLITE_TABLE_sources); + SQLite_init_table(db, "sources_source", SQLITE_TABLE_sources_source); + SQLite_init_table(db, "sources_patch", SQLITE_TABLE_sources_patch); +// SQLite_init_table(db, "sources_files_rel", SQLITE_TABLE_sources_files_rel); + SQLite_init_table(db, "sources_provided_rel", SQLITE_TABLE_sources_provided_rel); + SQLite_begin_transaction(db); + + currsource = ct->headersourcelist; + while (currsource != NULL) { + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO sources VALUES(NULL,?,%d,%d,?,?,?,%d,?,?,?,?,?,?,?,%d);", + currsource->altrepository, + currsource->epoch, + currsource->packager->id, + currsource->buildtime); + if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + } + + sqlite3_bind_text(stmt, 1, currsource->name, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, currsource->version, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, currsource->release, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 4, currsource->summary, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 5, currsource->group, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 6, currsource->description, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 7, currsource->url, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 8, currsource->license, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 9, currsource->arch, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 10, currsource->buildarchs, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 11, currsource->excludearch, -1, SQLITE_STATIC); + + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + return 3; + } + currsource->id = sqlite3_last_insert_rowid(db); + sqlite3_finalize(stmt); + + /* source */ + if (currsource->source) { + i=0; + while (currsource->source[i]) { + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO sources_source VALUES(NULL,%d,?);", + currsource->id); + + if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + } + sqlite3_bind_text(stmt, 1, currsource->source[i], -1, SQLITE_STATIC); + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + return 3; + } + sqlite3_finalize(stmt); + i++; + } + } + + /* patch */ + if (currsource->patch) { + i=0; + while (currsource->patch[i]) { + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO sources_patch VALUES(NULL,%d,?);", + currsource->id); + + if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + } + sqlite3_bind_text(stmt, 1, currsource->patch[i], -1, SQLITE_STATIC); + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + return 3; + } + sqlite3_finalize(stmt); + i++; + } + } + + /* sources <-> provided relations (buildrequirements) */ +/* for (i=0; iprovidecount; i++) { + snprintf(sqlite3_query, PATH_MAX, "INSERT INTO sources_provided_rel VALUES(NULL,%d,%d);", + currsource->id, + currsource->required[i]->id); + + if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + } + if (sqlite3_step(stmt) != SQLITE_DONE) { + fprintf(stderr, "ERROR: SQLite: (%s) %s\n", sqlite3_query, sqlite3_errmsg(db)); + return 3; + } + sqlite3_finalize(stmt); + }*/ + + currsource = currsource->next; + } + + SQLite_commit_transaction(db); + +/* if (sqlite3_finalize(stmt)) { + fprintf(stderr, "ERROR: SQLite: (%s) %s", sqlite3_query, sqlite3_errmsg(db)); + return 4; + }*/ +} + +int +generateSQLite(struct configTag *ct) +{ + sqlite3 *db = NULL; + char dbname[PATH_MAX]; + + snprintf(dbname, PATH_MAX, "/tmp/%s.db", ct->tag); + unlink(dbname); + + if (sqlite3_open_v2(dbname, &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, NULL)) { + if (db) sqlite3_close(db); + return 1; + } + + generateSQLite_packager(db); + generateSQLite_files(ct, db); + generateSQLite_provided(ct, db); + generateSQLite_sources(ct, db); + + sqlite3_close(db); + return 0; +} diff --git a/src/distromatic.c b/src/distromatic.c index 9ced6ae..267d65c 100644 --- a/src/distromatic.c +++ b/src/distromatic.c @@ -74,6 +74,7 @@ #include "requirelist.h" #include "functions.h" #include "rpmfunctions.h" +#include "backend-sqlite3.h" #define MODE_DATA_TABLES 1 #define MODE_FIND_DEPS 2 @@ -82,6 +83,7 @@ #define MODE_GENSRCPKGLIST 16 #define MODE_GENBUILDINFO 32 #define MODE_GENPKGLIST 64 +#define MODE_SQLITE3 128 static void program_usage(int exit_code); static void program_version(void); @@ -136,6 +138,7 @@ static const char *helpmsg[] = { " --genhtml generate HTML code for repository", " --genpkglist generate binary packages list with version and size", " --gensrcpkglist generate a source packages list with version", +" --gensqlite3 dump data to an SQLite3 database (EXPERIMENTAL)", " --find-deps find dependencies for given package name", " --changelog print changelog for given package name", " --changelogsince print changelog for all packages since given date", @@ -1066,6 +1069,7 @@ main(int argc, char *argv[]) { "genhtml", no_argument, 0, 0 }, { "genpkglist", no_argument, 0, 0 }, { "gensrcpkglist", no_argument, 0, 0 }, + { "gensqlite3", no_argument, 0, 0 }, { "arch", required_argument, 0, 'a' }, { "conf", required_argument, 0, 'c' }, { "help", no_argument, 0, 'h' }, @@ -1113,6 +1117,11 @@ main(int argc, char *argv[]) genheader_mode |= GENHEADER_CHANGELOG | GENHEADER_STATS; recursive_mode = 1; name = NULL; + } else if (!strcmp(longopts[longindex].name, "gensqlite3")) { + mode |= MODE_SQLITE3; + genheader_mode |= GENHEADER_CHANGELOG | GENHEADER_STATS; + recursive_mode = 1; + name = NULL; } else if (!strcmp(longopts[longindex].name, "gensrcpkglist")) { mode |= MODE_GENSRCPKGLIST; name = NULL; @@ -1310,6 +1319,7 @@ main(int argc, char *argv[]) logmsg(LOG_DEBUG,"generateHTMLFiles - done"); } + if (mode & MODE_GENPKGLIST) { /* currheaderlist = headerlist; */ generatePkgList(configtag, i); @@ -1368,6 +1378,13 @@ main(int argc, char *argv[]) } } // if (genheader_mode) + if (mode & MODE_SQLITE3) { + printf("Generating sqlite db...\n"); + logmsg(LOG_DEBUG,"generateSQLite - start"); + generateSQLite(configtag); + logmsg(LOG_DEBUG,"generateSQLite - done"); + } + if (genheader_mode & GENHEADER_STATS) { if (!quietmode) fprintf(stdout, " - generating source statistics...\n"); diff --git a/src/headerlist.c b/src/headerlist.c index 487098b..b0cef91 100644 --- a/src/headerlist.c +++ b/src/headerlist.c @@ -402,6 +402,8 @@ createProvidedListIndex(struct providedList* *idx) { } +static long providedListId = 0; + struct providedList* findOrCreateProvidedListEntry(struct providedList* *idx, char* findname, int create) @@ -451,6 +453,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx, newprovided->numversions = 0; newprovided->buildprovider = NULL; newprovided->name = strdup(findname); + newprovided->id = ++providedListId; return newprovided; } return currprovided; @@ -459,6 +462,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx, struct fileTree* findOrCreateFileTreeBrother(struct fileTree* *first,char* findname) { + static long id = 0; struct fileTree *newdir, *currdir, *prevdir = NULL; int c = 0; @@ -477,6 +481,7 @@ findOrCreateFileTreeBrother(struct fileTree* *first,char* findname) newdir->provider = NULL; newdir->numproviders = 0; newdir->parent = NULL; + newdir->id = id++; if (prevdir) { newdir->next = prevdir->next; diff --git a/src/include/backend-sqlite3.h b/src/include/backend-sqlite3.h new file mode 100644 index 0000000..5c283b2 --- /dev/null +++ b/src/include/backend-sqlite3.h @@ -0,0 +1,14 @@ +/* + * distromatic - tool for RPM based repositories + * + * Copyright (C) 2013 by Silvan Calarco + */ + +#ifndef BACKENDSQLITE3_H +#define BACKENDSQLITE3_H + +#include "distromatic.h" + +int generateSQLite(struct configTag *); + +#endif // BACKENDSQLITE3_H diff --git a/src/include/changelog.h b/src/include/changelog.h index bddc6e9..6e79b57 100644 --- a/src/include/changelog.h +++ b/src/include/changelog.h @@ -47,6 +47,7 @@ struct Packager { int changes_count; int packages_count; struct Packager *next; + long id; }; struct Packager *firstPackager(void); diff --git a/src/include/headerlist.h b/src/include/headerlist.h index 09608a2..ad98daa 100644 --- a/src/include/headerlist.h +++ b/src/include/headerlist.h @@ -33,6 +33,7 @@ struct providedList { char **version; int buildpriority; struct providedList *next; + long id; }; struct fileTree { @@ -42,6 +43,7 @@ struct fileTree { struct fileTree *parent; int numproviders; struct headerList **provider; + long id; }; struct warningList { @@ -123,6 +125,7 @@ struct headerSourceList { struct headerList *firstchild[ARCHS_MAX]; struct warningList *firstwarning; struct headerSourceList *updatingparent; + long id; }; struct headerStats {