From babfe854ea5309bd4bdb5e9ad7d7e681942cbd99 Mon Sep 17 00:00:00 2001 From: Silvan Calarco Date: Sun, 5 May 2013 17:35:01 +0200 Subject: [PATCH] Added per-arch multithreaded support for scanning binary packages --- src/Makefile.am | 2 +- src/distromatic.c | 198 +++++++++++++++++++------------------ src/headerlist.c | 66 ++++++++----- src/include/headerlist.h | 4 +- src/include/rpmfunctions.h | 5 +- src/rpmfunctions.c | 18 ++-- 6 files changed, 162 insertions(+), 131 deletions(-) diff --git a/src/Makefile.am b/src/Makefile.am index 3af4073..3234d97 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -17,7 +17,7 @@ AM_CPPFLAGS = -I$(top_srcdir)/src/include -I$(top_builddir)/src/include -LDFLAGS += $(SQLITE_LIBS) +LDFLAGS += $(SQLITE_LIBS) -lpthread bin_PROGRAMS = distromatic diff --git a/src/distromatic.c b/src/distromatic.c index 9663a69..6f15740 100644 --- a/src/distromatic.c +++ b/src/distromatic.c @@ -24,6 +24,7 @@ #include #include #include +#include #if HAVE_UNISTD_H # include @@ -121,6 +122,12 @@ PROGRAMNAME " version " PROGRAMVERSION, (char *)0 }; +unsigned int quietmode = 0; +unsigned int genheader_mode = GENHEADER_BASE + GENHEADER_REQUIREMENTS; +unsigned int mode = 0; +unsigned int recursive_mode = 0; +unsigned int obsolete_packages = 1; + /* *INDENT-OFF* */ static const char *freelicense[] = { "This is free software; see the source for copying conditions. There is NO", @@ -324,7 +331,7 @@ handleObsoletedPackages(struct configTag *ct, int archidx) while (currheader) { for (j = 0; j < currheader->obsoletecount; j++) { prov=findOrCreateProvidedListEntry((struct providedList**) &ct->providedlist_idx[archidx], - currheader->obsoletename[j],0); + currheader->obsoletename[j],0,archidx); if (prov) { for (i = 0; i < prov->numproviders; i++) { if (strcmp(currheader->name,prov->provider[i]->name)) { @@ -431,7 +438,7 @@ resolveFirstLevelDependencies(struct configTag *ct, int archidx) currheader->require[i]->resolved=NULL; } else if (strncmp("rpmlib(",currheader->require[i]->name,7) != 0) { provided=findOrCreateProvidedListEntry((struct providedList**) &ct->providedlist_idx[archidx], - currheader->require[i]->name,1); + currheader->require[i]->name,1,archidx); if (provided->numproviders == 0) { // check if require[i]->name requirement is met scanheader = ct->headerlist[archidx]; @@ -635,7 +642,7 @@ resolveFirstLevelSourceDependencies(struct configTag *ct, int archidx) }*/ if (strncmp("rpmlib(",currsourceheader->require[i]->name,7) != 0) { provided=findOrCreateProvidedListEntry((struct providedList**) &ct->providedlist_idx[archidx], - currsourceheader->require[i]->name,1); + currsourceheader->require[i]->name,1,archidx); if (provided->numbuildproviders == 0) { // check if require[i]->name requirement is met scanheader = ct->headerlist[archidx]; @@ -1036,6 +1043,88 @@ read_configuration(const char *confFile, const char *tag) return 0; } +void *threadArchScan(void* arg) { + int arch = *(int *)arg; + + if (!quietmode) + fprintf(stdout, "%s: scanning binary packages...\n",configtag->arch[arch]); + else + logmsg(LOG_MARK, "%s binary packages check for %s:", configtag->arch[arch],configtag->tag); + + if (generateHeaderList(configtag,arch)) { + fprintf(stderr, + "Fatal error: could not generate header list\n"); + exit(1); + } + + if (genheader_mode & GENHEADER_REQUIREMENTS) { + + if (obsolete_packages) { + if (!quietmode) fprintf(stdout, "%s: handling obsoleted packages...\n",configtag->arch[arch]); + if (handleObsoletedPackages(configtag,arch)) { + fprintf(stderr, + "Fatal error: maximum number of multiple providing packages reached. Aborting."); + exit(1); + } + } + + if (!quietmode) fprintf(stdout, "%s: resolving dependencies...\n",configtag->arch[arch]); + if (resolveFirstLevelDependencies(configtag, arch)) { + fprintf(stderr, + "Fatal error: could not generate first level requires table\n"); + exit(1); + } + + if (arch == 0) /* fixme? */ { + if (!quietmode) fprintf(stdout, "%s: resolving source dependencies\n",configtag->arch[arch]); + if (resolveFirstLevelSourceDependencies(configtag, arch)) { + fprintf(stderr, + "Fatal error: could not generate first level requires table\n"); + exit(1); + } + } + + if (recursive_mode) { + if (!quietmode) fprintf(stdout, "%s: resolving recursive dependencies...\n",configtag->arch[arch]); + + if (resolveRecursiveDependencies + (configtag->headerlist[arch])) { + fprintf(stderr,"Fatal error: could not resolve recursive dependencies\n"); + exit(1); + } + } + } + + if (mode & MODE_DATA_TABLES) { + if (!quietmode) fprintf(stdout,"%s: writing dependencies table...\n",configtag->arch[arch]); + print_datatables(configtag,arch); + } + + if (genheader_mode & GENHEADER_STATS) { + if (!quietmode) fprintf(stdout,"%s: generating statistics...\n",configtag->arch[arch]); + + logmsg(LOG_DEBUG,"%s: generateHeaderStats - start",configtag->arch[arch]); + if (generateHeaderStats(configtag, arch)) { + fprintf(stderr,"Fatal error: could not generate statistic for headers\n"); + exit(1); + } + logmsg(LOG_DEBUG,"%s: generateHeaderStats - done",configtag->arch[arch]); + } + + if (mode & MODE_HTML) { +// printf("Generating HTML files for binary RPMs...\n"); + logmsg(LOG_DEBUG,"%s: generateHTMLFiles - start",configtag->arch[arch]); + generateHTMLFiles(configtag, arch); + logmsg(LOG_DEBUG,"%s: generateHTMLFiles - done",configtag->arch[arch]); + } + + if (mode & MODE_GENPKGLIST) { + /* currheaderlist = headerlist; */ + generatePkgList(configtag, arch); + } + +} + int main(int argc, char *argv[]) { @@ -1046,12 +1135,8 @@ main(int argc, char *argv[]) struct headerList *currheaderlist; struct headerSourceList *currheadersourcelist = NULL; - unsigned int mode = 0; - unsigned int recursive_mode = 0; - unsigned int genheader_mode = GENHEADER_BASE + GENHEADER_REQUIREMENTS; - unsigned int quietmode = 0; - unsigned int obsolete_packages = 1; - int i,hasbuilds[ARCHS_MAX]; + int i,hasbuilds[ARCHS_MAX],ptharg[ARCHS_MAX]; + pthread_t pth[ARCHS_MAX]; char warning[PATH_MAX]; time_t start_time, stop_time; @@ -1238,95 +1323,14 @@ main(int argc, char *argv[]) } for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { + ptharg[i]=i; + pthread_create(&pth[i],NULL,threadArchScan,&ptharg[i]); + } // archs threads loop - if (!quietmode) - fprintf(stdout, "Scanning binary packages for %s arch...\n",configtag->arch[i]); - else - logmsg(LOG_MARK, "%s binary packages check for %s:", configtag->arch[i],configtag->tag); - - if (generateHeaderList(configtag,i)) { - fprintf(stderr, - "Fatal error: could not generate header list\n"); - exit(1); - } - - - if (genheader_mode & GENHEADER_REQUIREMENTS) { - - if (obsolete_packages) { - if (!quietmode) - fprintf(stdout, " - handling obsoleted packages...\n"); - if (handleObsoletedPackages(configtag,i)) { - fprintf(stderr, - "Fatal error: maximum number of multiple providing packages reached. Aborting."); - exit(1); - } - } - - if (!quietmode) - fprintf(stdout, " - resolving dependencies...\n"); - if (resolveFirstLevelDependencies(configtag, i)) { - fprintf(stderr, - "Fatal error: could not generate first level requires table\n"); - exit(1); - } - - if (i == 0) /* fixme? */ { - if (!quietmode) - fprintf(stdout, " - resolving source dependencies\n"); - if (resolveFirstLevelSourceDependencies(configtag, i)) { - fprintf(stderr, - "Fatal error: could not generate first level requires table\n"); - exit(1); - } - } - - if (recursive_mode) { - if (!quietmode) - fprintf(stdout, " - resolving recursive dependencies...\n"); - - if (resolveRecursiveDependencies - (configtag->headerlist[i])) { - fprintf(stderr, - "Fatal error: could not resolve recursive dependencies\n"); - exit(1); - } - } - } - - if (mode & MODE_DATA_TABLES) { - if (!quietmode) fprintf(stdout, " - writing dependencies table...\n"); - print_datatables(configtag,i); - } - - if (genheader_mode & GENHEADER_STATS) { - if (!quietmode) - fprintf(stdout, " - generating statistics...\n"); - - logmsg(LOG_DEBUG,"generateHeaderStats - start"); - if (generateHeaderStats(configtag, i)) { - fprintf(stderr, - "Fatal error: could not generate statistic for headers\n"); - exit(1); - } - logmsg(LOG_DEBUG,"generateHeaderStats - done"); - } - - if (mode & MODE_HTML) { -// printf("Generating HTML files for binary RPMs...\n"); - logmsg(LOG_DEBUG,"generateHTMLFiles - start"); - generateHTMLFiles(configtag, i); - logmsg(LOG_DEBUG,"generateHTMLFiles - done"); - } - - - if (mode & MODE_GENPKGLIST) { - /* currheaderlist = headerlist; */ - generatePkgList(configtag, i); - } - - } // archs loop - + for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { + pthread_join(pth[i], NULL); + } + if (!passed_arch) { // can't do missing builds and ports check in single arch mode if (!quietmode) fprintf(stdout, "Checking for SRPMS with no builds and missing ports...\n"); diff --git a/src/headerlist.c b/src/headerlist.c index 3a7d46a..3f3a066 100644 --- a/src/headerlist.c +++ b/src/headerlist.c @@ -26,6 +26,12 @@ # include #endif +#if RPM_VERSION >= 0x040100 +#include +#else +#define rpmReadPackageFile(a,b,c,d) rpmReadPackageHeader(b,d,0,NULL,NULL) +#endif + #if HAVE_STRING_H # if !STDC_HEADERS && HAVE_MEMORY_H # include @@ -56,10 +62,10 @@ # define memcpy(d, s, n) bcopy ((s), (d), (n)) #endif -static int rpmselector(const struct dirent *entry); -static int sourcerpmselector(const struct dirent *entry); +int rpmselector(const struct dirent *entry); +int sourcerpmselector(const struct dirent *entry); -static const int bufsize = 1024; +const int bufsize = 1024; struct warningList* addWarning(struct headerSourceList *pkg, char* text) { @@ -332,7 +338,7 @@ generateHeaderSourceStats(struct configTag *ct) /* * this function should filter only RPM files * fixme: it currently just filters files from other entities */ -static int +int rpmselector(const struct dirent *entry) { char *ptr; @@ -354,7 +360,7 @@ rpmselector(const struct dirent *entry) return 1; } -static int +int sourcerpmselector(const struct dirent *entry) { char *ptr; @@ -371,7 +377,7 @@ sourcerpmselector(const struct dirent *entry) return 1; } -static long providedListCount = 0; +long providedListCount[ARCHS_MAX]; void cleanProvidedListIndex(struct configTag *ct, int arch) @@ -381,17 +387,17 @@ cleanProvidedListIndex(struct configTag *ct, int arch) for (i = 0; i < PROVIDEDLIST_IDX_SIZE; i++) ct->providedlist_idx[arch][i]=NULL; - providedListCount = 0; + providedListCount[arch] = 0; ct->filetree[arch]=NULL; } void -createProvidedListIndex(struct providedList* *idx) { - int i, step = providedListCount/PROVIDEDLIST_IDX_SIZE; +createProvidedListIndex(struct providedList* *idx, int arch) { + int i, step = providedListCount[arch]/PROVIDEDLIST_IDX_SIZE; int curridx = 1; struct providedList *currprovided = idx[0]; - if (providedListCount > PROVIDEDLIST_IDX_SIZE) { + if (providedListCount[arch] > PROVIDEDLIST_IDX_SIZE) { while (currprovided && curridx < PROVIDEDLIST_IDX_SIZE) { for (i = 0; i < step; i++) currprovided = currprovided->next; @@ -406,7 +412,7 @@ static long providedListId = 0; struct providedList* findOrCreateProvidedListEntry(struct providedList* *idx, - char* findname, int create) + char* findname, int create, int arch) { struct providedList *newprovided, *currprovided, *prevprovided = NULL; int c = 0, i; @@ -438,7 +444,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx, if ((!currprovided) || (c > 0)) { if (!create) return NULL; - providedListCount++; + providedListCount[arch]++; newprovided = malloc(sizeof(struct providedList)); if (prevprovided) { newprovided->next = prevprovided->next; @@ -554,6 +560,11 @@ addToSourceHeaderList(struct headerSourceList **headersourcelist, struct configT char **requireversion, **requirename; uint_32 *requireflags; char warning[PATH_MAX]; +#if RPM_VERSION >= 0x040100 + rpmts ts = rpmtsCreate(); +#else + rpmts ts = NULL; +#endif if (altrepository == ct->repository_level) { scanpath = ct->repository_source_dir; @@ -581,7 +592,7 @@ addToSourceHeaderList(struct headerSourceList **headersourcelist, struct configT } strcpy(&filepath[strlen(scanpath)], namelist[cnt]->d_name); - if (getHeader(filepath, &h)) { + if (getHeader(&ts, filepath, &h)) { errstr = strerror(errno); logmsg(LOG_WARNING, "%s: unable to read header (%s); skipping.",namelist[cnt]->d_name, errstr); @@ -726,6 +737,7 @@ addToSourceHeaderList(struct headerSourceList **headersourcelist, struct configT } // for free(namelist); + rpmtsFree(ts); return 0; } @@ -765,9 +777,9 @@ generateSourceHeaderList(struct configTag *ct, int mode) int addToHeaderList(struct configTag *ct, const char *scanpath, /*int mode,*/ - const char* scantag, - const char* altscanpath[ALT_REPS_MAX], - const char* altscantag[ALT_REPS_MAX], + const char *scantag, + const char *altscanpath[ALT_REPS_MAX], + const char *altscantag[ALT_REPS_MAX], int arch) { @@ -791,8 +803,12 @@ addToHeaderList(struct configTag *ct, **requirename, **requireversion, **basename, **dirname, **newversion; const char* errstr; - uint_32 *requireflags, *obsoleteflags, *provideflags; +#if RPM_VERSION >= 0x040100 + rpmts ts = rpmtsCreate(); +#else + rpmts ts = NULL; +#endif currheaderlist = ct->headerlist[arch]; @@ -885,7 +901,9 @@ addToHeaderList(struct configTag *ct, /* process package */ logmsg(LOG_DEBUG, "getting header for %s", filepath); - if (getHeader(filepath, &h)) { + sem_wait(&rpm_mutex); + if (getHeader(&ts, filepath, &h)) { + sem_post(&rpm_mutex); errstr = strerror(errno); logmsg(LOG_WARNING, "%s: unable to read header (%s); skipping.",filename, errstr); @@ -907,6 +925,7 @@ addToHeaderList(struct configTag *ct, } memset(newheaderlist, 0, sizeof(struct headerList)); getPackageInfoIntoHeaderList(h, newheaderlist); + sem_post(&rpm_mutex); if (!newheaderlist->sourcename) { logmsg(LOG_WARNING, @@ -976,7 +995,7 @@ addToHeaderList(struct configTag *ct, newheaderlist->provided = malloc(sizeof(struct providedList*)*providecount); for (i=0; i < providecount; i++) { provided = findOrCreateProvidedListEntry((struct providedList **) &(ct->providedlist_idx[arch]), - providename[i],1); + providename[i],1,arch); newheaderlist->provided[i]=provided; if (provided && (provided->numproviders == 0)) { provided->flags=provideflags[i]; @@ -986,7 +1005,7 @@ addToHeaderList(struct configTag *ct, provided->numversions++; provided->version=malloc(sizeof(char *)); provided->version[0]=strdup(provideversion[i]); - } else if (provided->numproviders > 0) { + } else if (provided && (provided->numproviders > 0)) { provided->numproviders++; newprovider=malloc(sizeof(struct headerList*)*provided->numproviders); for (j = 0; j < provided->numproviders-1; j++) { @@ -996,11 +1015,11 @@ addToHeaderList(struct configTag *ct, free(provided->provider); provided->provider=newprovider; - /* check if version is already provided */ + // check if version is already provided for (j = 0; j < provided->numversions; j++) { if (!strcmp(provided->version[j],provideversion[i])) break; } - /* if not add to the list of versions */ + // if not add to the list of versions if (j == provided->numversions) { provided->numversions++; newversion=malloc(sizeof(char *)*provided->numversions); @@ -1074,8 +1093,9 @@ addToHeaderList(struct configTag *ct, free(altnamelist[altrepository-1][altcnt[altrepository-1]-1]); } else free(namelist[cnt-1]); } // main while + rpmtsFree(ts); free(namelist); - createProvidedListIndex((struct providedList **)&(ct->providedlist_idx[arch])); + createProvidedListIndex((struct providedList **)&(ct->providedlist_idx[arch]), arch); return 0; } diff --git a/src/include/headerlist.h b/src/include/headerlist.h index fda924c..75b2c7d 100644 --- a/src/include/headerlist.h +++ b/src/include/headerlist.h @@ -167,7 +167,7 @@ generateHeaderSourceStats(struct configTag *ct); struct providedList* findOrCreateProvidedListEntry(struct providedList **idx, - char* findname, int create); + char* findname, int create, int arch); struct fileTree* findOrCreateFileTreeBrother(struct fileTree* *first,char* findname); @@ -179,7 +179,7 @@ void cleanProvidedListIndex(struct configTag *ct, int arch); void -createProvidedListIndex(struct providedList* *idx); +createProvidedListIndex(struct providedList* *idx, int arch); int generateHeaderList( struct configTag *ct, diff --git a/src/include/rpmfunctions.h b/src/include/rpmfunctions.h index c0b224a..974bf35 100644 --- a/src/include/rpmfunctions.h +++ b/src/include/rpmfunctions.h @@ -12,6 +12,7 @@ # include #endif +#include #include "headerlist.h" #define HEADERS_BUFFER_SIZE 2000000 @@ -30,7 +31,7 @@ long long headerGetUIntEntry(Header h, const int tag); void *headerGetUIntArrayEntry(Header h, const int tag, int *count); -int getHeader(char *headerFile, Header * h); +int getHeader(rpmts *ts, char *headerFile, Header * h); int getPackageRequires( Header h, char ***requirename, uint_32 **requireflags, @@ -55,4 +56,6 @@ int scanrpmnamecmp(const struct dirent **f1, const struct dirent **f2); char* printrpmversion(char *s, int bufsize, long epoch, char *version, char *release); +sem_t rpm_mutex; + #endif // RPMFUNCTIONS_H diff --git a/src/rpmfunctions.c b/src/rpmfunctions.c index aad9503..d249874 100644 --- a/src/rpmfunctions.c +++ b/src/rpmfunctions.c @@ -89,10 +89,10 @@ unsigned int checkVersionWithFlags(const char* cmp1, uint_32 flags, const char* /* * get header from a file descriptor of a YUM compressed header file */ int -getHeader(char *headerFile, Header * h) +getHeader(rpmts* ts, char *headerFile, Header * h) { char buffer[HEADERS_BUFFER_SIZE]; - int len; + int len,lock=0; /* check if file is a compressed header or a RPM/SRPM */ if (!strcmp(&headerFile[strlen(headerFile) - 4], ".hdr")) { @@ -124,16 +124,19 @@ getHeader(char *headerFile, Header * h) int rc; #if RPM_VERSION >= 0x040100 - rc = rpmReadPackageFile(ts, fd, headerFile, h); - if (rc != RPMRC_OK && rc != RPMRC_NOTTRUSTED && rc != RPMRC_NOKEY) { - /* fprintf(stderr, "Error: Failed reading file %s", headerFile); */ + rc = rpmReadPackageFile(*ts, fd, headerFile, h); #else rc = rpmReadPackageHeader(fd, h, NULL, NULL, NULL); +#endif + fdClose(fd); +#if RPM_VERSION >= 0x040100 + if (rc != RPMRC_OK && rc != RPMRC_NOTTRUSTED && rc != RPMRC_NOKEY) { +// fprintf(stderr, "Error: Failed reading file %s\n", headerFile); +#else if (rc != 0) { #endif return 1; } - fdClose(fd); } return 0; @@ -366,7 +369,8 @@ char* printrpmversion(char *s, int bufsize, long epoch, char *version, char *rel void rpminit() { #if RPM_VERSION >= 0x040100 rpmReadConfigFiles(NULL, NULL); - ts = rpmtsCreate(); +// ts = rpmtsCreate(); // rpmtsSetVSFlags(ts, (rpmVSFlags_e)-1); #endif + sem_init(&rpm_mutex, 0, 1); }