From f804316c02da46bccd11e2e137cc1bce18828507 Mon Sep 17 00:00:00 2001 From: Silvan Calarco Date: Tue, 16 Feb 2021 14:44:40 +0100 Subject: [PATCH] repodata scan: added support for files provides; added -p option to enable parallel per-arch processing (default: disabled) --- src/distromatic.c | 45 +++++++++++----- src/headerlist.c | 128 +++++++++++++++++++++++++++++++--------------- 2 files changed, 118 insertions(+), 55 deletions(-) diff --git a/src/distromatic.c b/src/distromatic.c index 767c7c2..f24fd6c 100644 --- a/src/distromatic.c +++ b/src/distromatic.c @@ -154,11 +154,12 @@ static const char *helpmsg[] = { " --changelogsince print changelog for all packages since given date", "", "Options:", -" -t, --tag use repository identified by tag in config file", " -a, --arch specify a target architecture (default: " DEFAULT_ARCH ")", +" -d, --debug display debug level messages", " -c, --conf specify configuration file (default: " DEFAULT_CONFIGFILE ")", " -q, --quiet suppress all messages excluding warnings and errors", -" -d, --debug display debug level messages", +" -p, --parallel enable parallel execution of per-arch processing", +" -t, --tag use repository identified by tag in config file", " -v, --version display version and exit", "", "Examples:", @@ -942,13 +943,14 @@ main(int argc, char *argv[]) struct rebuildList *currrebuild; int i = 0,hasbuilds[ARCHS_MAX],ptharg[ARCHS_MAX]; + int parallel = 0; pthread_t pth[ARCHS_MAX]; char warning[8096]; time_t start_time, stop_time; /* options args for 'getopt_long()' */ - static const char *optstring = "+a:c:t:qhvd"; + static const char *optstring = "+a:c:t:qhvdp"; static struct option longopts[] = { { "deps-table", no_argument, 0, 0 }, { "data-tables", no_argument, 0, 0 }, @@ -971,6 +973,7 @@ main(int argc, char *argv[]) { "tag", required_argument, 0, 't' }, { "version", no_argument, 0, 'v' }, { "debug", no_argument, 0, 'd' }, + { "parallel", no_argument, 0, 'p' }, { 0, 0, 0, 0 } }; @@ -1065,6 +1068,9 @@ main(int argc, char *argv[]) log_debug_set(1); logmsg(LOG_DEBUG,"debug logging enabled"); break; + case 'p': + parallel = 1; + break; default: fprintf(stderr, "Fatal error while parsing command line arguments\n"); @@ -1154,22 +1160,35 @@ main(int argc, char *argv[]) } warning[0]=0; - for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { - snprintf(&warning[strlen(warning)],sizeof(warning)-strlen(warning)," %s", configtag->arch[i]); + if (parallel) { + for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { + snprintf(&warning[strlen(warning)],sizeof(warning)-strlen(warning)," %s", configtag->arch[i]); + } + if (!quietmode) + fprintf(stdout, "Scanning binary packages for archs:%s...\n",warning); + else + logmsg(LOG_MARK, "Binary packages check for archs:%s", warning); } - if (!quietmode) - fprintf(stdout, "Scanning binary packages for archs:%s...\n",warning); - else - logmsg(LOG_MARK, "Binary packages check for archs:%s", warning); for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { ptharg[i]=i; - pthread_create(&pth[i],NULL,threadArchScan,&ptharg[i]); + if (!parallel) { + if (!quietmode) + fprintf(stdout, "Scanning binary packages for arch %s...\n", + configtag->arch[i]); + else + logmsg(LOG_MARK, "Binary packages check for arch %s", + configtag->arch[i]); + threadArchScan(&ptharg[i]); + } + else + pthread_create(&pth[i],NULL,threadArchScan,&ptharg[i]); } // archs threads loop - for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { - pthread_join(pth[i], NULL); - } + if (parallel) + for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { + pthread_join(pth[i], NULL); + } if (!passed_arch) { // can't do missing builds and ports check in single arch mode if (!quietmode) diff --git a/src/headerlist.c b/src/headerlist.c index bd032ef..7b44b7a 100644 --- a/src/headerlist.c +++ b/src/headerlist.c @@ -452,8 +452,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx, } currprovided = idx[idxbot]; - while (currprovided && - ((c=strcmp(currprovided->name,findname)) < 0)) { + while (currprovided && ((c=strcmp(currprovided->name,findname)) < 0)) { prevprovided = currprovided; currprovided = currprovided->next; } @@ -482,7 +481,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx, } struct fileTree* -findOrCreateFileTreeBrother(struct fileTree* *first,char* findname, int arch) +findOrCreateFileTreeBrother(struct fileTree* *first, char* findname, int arch) { static long id[ARCHS_MAX] = { 0, 0, 0, 0, 0}; struct fileTree *newdir, *currdir, *prevdir = NULL; @@ -515,13 +514,12 @@ findOrCreateFileTreeBrother(struct fileTree* *first,char* findname, int arch) newdir->name = strdup(findname); return newdir; } - return currdir; } struct fileTree* -findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch) +findOrCreateFileTreeEntry(struct fileTree* *first, char* findname, int arch) { struct fileTree *currdir,*prevdir=NULL; char *pstart,*pend; @@ -529,8 +527,8 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch) int l; currdir = *first; - pstart = findname+1; /* skip trailing slash */ - pend = findname+1; + pstart = &(findname[1]); /* skip trailing slash */ + pend = &(findname[1]); l = strlen(findname); while (pend-findname < l) { @@ -542,7 +540,7 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch) if (prevdir) { currdir = findOrCreateFileTreeBrother(&prevdir->firstchild,f,arch); - currdir->parent = prevdir; + currdir->parent = prevdir; } else currdir = findOrCreateFileTreeBrother(first,f,arch); @@ -552,8 +550,6 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch) prevdir = currdir; pstart=pend+1; } -// fprintf(stderr,"%s (%d) firstchild:%d\n",currdir->name,prevdir,currdir->firstchild); - return currdir; } @@ -859,8 +855,8 @@ void getXMLPackageFiles(xmlNode *parent, uint **type, char ***path, int *count) *path = NULL; return; } - *type = malloc(sizeof(int*) * (*count)); - *path = malloc(sizeof(char*) * (*count)+1); + *type = malloc(sizeof(uint*) * (*count)); + *path = malloc(sizeof(char*) * (*count+1)); int i = 0; for (xmlNode *entry=parent->children; entry; entry=entry->next) { @@ -1033,9 +1029,13 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) { // Use repodata //n = atoi((char*)findXMLPropertyByName(xmlDocGetRootElement(repodata->primary_doc), "packages")); //xmlNode* packageNode = findXMLPackageByName(xmlDocGetRootElement(repodata->other_doc), "firefox"); - xmlNode *package_node; - for (package_node = xmlDocGetRootElement(repodata->primary_doc)->children; - package_node; package_node = package_node->next) { + xmlNode *filelists_node = xmlDocGetRootElement(repodata->filelists_doc)->children; + xmlNode *other_node = xmlDocGetRootElement(repodata->other_doc)->children; + for (xmlNode *package_node = xmlDocGetRootElement(repodata->primary_doc)->children; + package_node; + package_node = package_node->next, + filelists_node = filelists_node->next, + other_node = other_node->next) { if (!strcmp((char*)package_node->name, "package")) { //print_element_names(package_node->children, 0); newheadersourcelist = malloc(sizeof(struct headerSourceList)); @@ -1099,7 +1099,7 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) { (char*)findXMLAttributeByName(psize, "package")); //hl->patch = headerGetStringArrayEntry(h, RPMTAG_PATCH, &count); - // XML requires + // Requires getXMLPackageNFV(findXMLPropertyByName(format, "requires"), &requirename, &requireflags, &requireversion, &requirecount); newheadersourcelist->require = malloc(requirecount * @@ -1113,16 +1113,14 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) { } newheadersourcelist->requirecount = requirecount; - // XML Files - xmlNode* filelistsNode = findXMLPackageByName(xmlDocGetRootElement( - repodata->filelists_doc), newheadersourcelist->name); - - getXMLPackageFiles(filelistsNode, &fileflags, &basenames, &filenamescount); + // Files + getXMLPackageFiles(filelists_node, &fileflags, &basenames, &filenamescount); newheadersourcelist->source = basenames; newheadersourcelist->filenamecount = 0; - addNewToSourceHeaderList(newheadersourcelist, ct, altrepository); // TODO: changelog + + addNewToSourceHeaderList(newheadersourcelist, ct, altrepository); } } cleanRepodata(repodata); @@ -1317,23 +1315,29 @@ int addNewToHeaderList(struct headerList **currheaderlist, return 0; } -char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) { +char* advanceXMLPackageNode(char* prevname, xmlNode **primary_node, + xmlNode **filelists_node) { char* currname; // check if already currname > prevname - if (*package_node && !strcmp((char*)(*package_node)->name, "package")) { + if (*primary_node && !strcmp((char*)(*primary_node)->name, "package")) { currname = strdup((char*) - findXMLPropertyByName(*package_node, "name")->children->content); - // package_node is already advanced + findXMLPropertyByName(*primary_node, "name")->children->content); + // primary_node is already advanced if (strcmp(prevname, currname) < 0) return currname; free(currname); } // nope: go to next package - if (*package_node) *package_node = (*package_node)->next; - while (*package_node && strcmp((char*)(*package_node)->name, "package")) - *package_node = (*package_node)->next; - if (*package_node) { + if (*primary_node) { + *primary_node = (*primary_node)->next; + *filelists_node = (*filelists_node)->next; + } + while (*primary_node && strcmp((char*)(*primary_node)->name, "package")) { + *primary_node = (*primary_node)->next; + *filelists_node = (*filelists_node)->next; + } + if (*primary_node) { currname = strdup((char*) - findXMLPropertyByName(*package_node, "name")->children->content); + findXMLPropertyByName(*primary_node, "name")->children->content); return currname; } return NULL; @@ -1345,7 +1349,7 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) { int generateHeaderList(struct configTag* ct, int arch, int incremental) { struct repoData* repodata[ALT_REPS_MAX + 1]; - xmlNode *package_node[ALT_REPS_MAX + 1]; + xmlNode *primary_node[ALT_REPS_MAX + 1], *filelists_node[ALT_REPS_MAX + 1]; char nextname[bufsize + 1]; char scanpath[bufsize + 1], *altscanpath[ALT_REPS_MAX]; @@ -1417,7 +1421,8 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) { if (repodata[0]) { for (i = 0; i <= alt_reps_num; i++) { - package_node[i] = xmlDocGetRootElement(repodata[i]->primary_doc)->children; + primary_node[i] = xmlDocGetRootElement(repodata[i]->primary_doc)->children; + filelists_node[i] = xmlDocGetRootElement(repodata[i]->filelists_doc)->children; } } else { n = scansdir(scanpath, &namelist, rpmselector, scanrpmnamecmp); @@ -1448,7 +1453,8 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) { char *newname = NULL; nextname[0] = '\0'; for (i = 0; i <= alt_reps_num; i++) { - newname = advanceXMLPackageNode(currname, &(package_node[i])); + newname = advanceXMLPackageNode(currname, &(primary_node[i]), + &(filelists_node[i])); if (newname) { if (nextname[0] == '\0' || strcmp(nextname, newname) >= 0) { strncpy(nextname, newname, bufsize); @@ -1477,25 +1483,25 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) { memset(newheaderlist, 0, sizeof(struct headerList)); newheaderlist->name = strdup((char*) - findXMLPropertyByName(package_node[altidx], "name")->children->content); - xmlNode *version = findXMLPropertyByName(package_node[altidx], "version"); + findXMLPropertyByName(primary_node[altidx], "name")->children->content); + xmlNode *version = findXMLPropertyByName(primary_node[altidx], "version"); newheaderlist->epoch = atoi((char*)findXMLAttributeByName(version, "epoch")); newheaderlist->version = strdup((char*)findXMLAttributeByName(version, "ver")); newheaderlist->release = strdup((char*)findXMLAttributeByName(version, "rel")); newheaderlist->summary = strdup((char*) - findXMLPropertyByName(package_node[altidx], "summary")->children->content); + findXMLPropertyByName(primary_node[altidx], "summary")->children->content); newheaderlist->arch = strdup((char*) - findXMLPropertyByName(package_node[altidx], "arch")->children->content); - if (findXMLPropertyByName(package_node[altidx], "description")->children) { + findXMLPropertyByName(primary_node[altidx], "arch")->children->content); + if (findXMLPropertyByName(primary_node[altidx], "description")->children) { newheaderlist->description = strdup((char*) - findXMLPropertyByName(package_node[altidx], "description")->children->content); + findXMLPropertyByName(primary_node[altidx], "description")->children->content); } - xmlNode *format = findXMLPropertyByName(package_node[altidx], "format"); + xmlNode *format = findXMLPropertyByName(primary_node[altidx], "format"); newheaderlist->group = strdup((char*) findXMLPropertyByName(format, "group")->children->content); newheaderlist->sourcename = strdup((char*) findXMLPropertyByName(format, "sourcerpm")->children->content); - xmlNode *psize = findXMLPropertyByName(package_node[altidx], "size"); + xmlNode *psize = findXMLPropertyByName(primary_node[altidx], "size"); newheaderlist->size = atoi( (char*)findXMLAttributeByName(psize, "package")); @@ -1552,6 +1558,7 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) { newheaderlist->obsoleteversion = obsoleteversion; newheaderlist->obsoletecount = obsoletecount; newheaderlist->altrepository = altrepository; + // Requires newheaderlist->require = malloc(requirecount * sizeof(struct Require *)); for (j=0; j < requirecount; j++) { newheaderlist->require[j] = malloc(sizeof(struct Require)); @@ -1561,6 +1568,43 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) { newheaderlist->require[j]->resolved = NULL; } newheaderlist->requirecount = requirecount; + // Files + getXMLPackageFiles(filelists_node[altidx], &fileflags, &basename, + &filenamecount); + newheaderlist->file = malloc(sizeof(struct fileTree*) * filenamecount); + newheaderlist->filenamecount = filenamecount; + for (j=0; jfile[j] = findOrCreateFileTreeEntry(&ct->filetree[arch], basename[j], arch); + free(basename[j]); + //newheaderlist->fileflags[j] = fileflags[j]; + //newheaderlist->fileuser[j] = findOrCreateFileUserListEntry(&ct->fileuserlist[arch], NULL, arch); + //newheaderlist->filegroup[j] = findOrCreateFileGroupListEntry(&ct->filegrouplist[arch], NULL, arch); + if (newheaderlist->file[j]->numproviders == 0) { + newheaderlist->file[j]->numproviders++; + newheaderlist->file[j]->provider=malloc(sizeof(struct headerList*)); + newheaderlist->file[j]->provider[0]=newheaderlist; + } else { + // check if the package is already providing it... + for (k = 0; k < newheaderlist->file[j]->numproviders; k++) { + if (newheaderlist->file[j]->provider[k] == newheaderlist) + break; + } + // if not add to the list of providers + if (k == newheaderlist->file[j]->numproviders) { + newheaderlist->file[j]->numproviders++; + newprovider=malloc( + sizeof(struct headerList*)*newheaderlist->file[j]->numproviders); + for (k = 0; k < newheaderlist->file[j]->numproviders-1; k++) { + newprovider[k]=newheaderlist->file[j]->provider[k]; + } + newprovider[newheaderlist->file[j]->numproviders-1] = newheaderlist; + free(newheaderlist->file[j]->provider); + newheaderlist->file[j]->provider=newprovider; + } + } + } + free(basename); + free(fileflags); } } else {