openmamba/distromatic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

repodata scan: added support for files provides; added -p option to enable parallel per-arch processing (default: disabled)

Browse Source
This commit is contained in:
Silvan Calarco 2021-02-16 14:44:40 +01:00
parent 1aa74444e2
commit f804316c02
2 changed files with 118 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/distromatic.c
View File

@ -154,11 +154,12 @@ static const char *helpmsg[] = {
" --changelogsince <mmddyy> print changelog for all packages since given date", " --changelogsince <mmddyy> print changelog for all packages since given date",
"", "",
"Options:", "Options:",
" -t, --tag use repository identified by tag in config file",
" -a, --arch specify a target architecture (default: " DEFAULT_ARCH ")", " -a, --arch specify a target architecture (default: " DEFAULT_ARCH ")",
" -d, --debug display debug level messages",
" -c, --conf <file> specify configuration file (default: " DEFAULT_CONFIGFILE ")", " -c, --conf <file> specify configuration file (default: " DEFAULT_CONFIGFILE ")",
" -q, --quiet suppress all messages excluding warnings and errors", " -q, --quiet suppress all messages excluding warnings and errors",
" -d, --debug display debug level messages", " -p, --parallel enable parallel execution of per-arch processing",
" -t, --tag use repository identified by tag in config file",
" -v, --version display version and exit", " -v, --version display version and exit",
"", "",
"Examples:", "Examples:",
@ -942,13 +943,14 @@ main(int argc, char *argv[])
struct rebuildList *currrebuild; struct rebuildList *currrebuild;
int i = 0,hasbuilds[ARCHS_MAX],ptharg[ARCHS_MAX]; int i = 0,hasbuilds[ARCHS_MAX],ptharg[ARCHS_MAX];
int parallel = 0;
pthread_t pth[ARCHS_MAX]; pthread_t pth[ARCHS_MAX];
char warning[8096]; char warning[8096];
time_t start_time, stop_time; time_t start_time, stop_time;
/* options args for 'getopt_long()' */ /* options args for 'getopt_long()' */
static const char *optstring = "+a:c:t:qhvd"; static const char *optstring = "+a:c:t:qhvdp";
static struct option longopts[] = { static struct option longopts[] = {
{ "deps-table", no_argument, 0, 0 }, { "deps-table", no_argument, 0, 0 },
{ "data-tables", no_argument, 0, 0 }, { "data-tables", no_argument, 0, 0 },
@ -971,6 +973,7 @@ main(int argc, char *argv[])
{ "tag", required_argument, 0, 't' }, { "tag", required_argument, 0, 't' },
{ "version", no_argument, 0, 'v' }, { "version", no_argument, 0, 'v' },
{ "debug", no_argument, 0, 'd' }, { "debug", no_argument, 0, 'd' },
{ "parallel", no_argument, 0, 'p' },
{ 0, 0, 0, 0 } { 0, 0, 0, 0 }
}; };
@ -1065,6 +1068,9 @@ main(int argc, char *argv[])
log_debug_set(1); log_debug_set(1);
logmsg(LOG_DEBUG,"debug logging enabled"); logmsg(LOG_DEBUG,"debug logging enabled");
break; break;
case 'p':
parallel = 1;
break;
default: default:
fprintf(stderr, fprintf(stderr,
"Fatal error while parsing command line arguments\n"); "Fatal error while parsing command line arguments\n");
@ -1154,22 +1160,35 @@ main(int argc, char *argv[])
} }
warning[0]=0; warning[0]=0;
for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { if (parallel) {
snprintf(&warning[strlen(warning)],sizeof(warning)-strlen(warning)," %s", configtag->arch[i]); for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
snprintf(&warning[strlen(warning)],sizeof(warning)-strlen(warning)," %s", configtag->arch[i]);
}
if (!quietmode)
fprintf(stdout, "Scanning binary packages for archs:%s...\n",warning);
else
logmsg(LOG_MARK, "Binary packages check for archs:%s", warning);
} }
if (!quietmode)
fprintf(stdout, "Scanning binary packages for archs:%s...\n",warning);
else
logmsg(LOG_MARK, "Binary packages check for archs:%s", warning);
for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
ptharg[i]=i; ptharg[i]=i;
pthread_create(&pth[i],NULL,threadArchScan,&ptharg[i]); if (!parallel) {
if (!quietmode)
fprintf(stdout, "Scanning binary packages for arch %s...\n",
configtag->arch[i]);
else
logmsg(LOG_MARK, "Binary packages check for arch %s",
configtag->arch[i]);
threadArchScan(&ptharg[i]);
}
else
pthread_create(&pth[i],NULL,threadArchScan,&ptharg[i]);
} // archs threads loop } // archs threads loop
for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) { if (parallel)
pthread_join(pth[i], NULL); for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
} pthread_join(pth[i], NULL);
}
if (!passed_arch) { // can't do missing builds and ports check in single arch mode if (!passed_arch) { // can't do missing builds and ports check in single arch mode
if (!quietmode) if (!quietmode)

128
src/headerlist.c
View File

@ -452,8 +452,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx,
} }
currprovided = idx[idxbot]; currprovided = idx[idxbot];
while (currprovided && while (currprovided && ((c=strcmp(currprovided->name,findname)) < 0)) {
((c=strcmp(currprovided->name,findname)) < 0)) {
prevprovided = currprovided; prevprovided = currprovided;
currprovided = currprovided->next; currprovided = currprovided->next;
} }
@ -482,7 +481,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx,
} }
struct fileTree* struct fileTree*
findOrCreateFileTreeBrother(struct fileTree* *first,char* findname, int arch) findOrCreateFileTreeBrother(struct fileTree* *first, char* findname, int arch)
{ {
static long id[ARCHS_MAX] = { 0, 0, 0, 0, 0}; static long id[ARCHS_MAX] = { 0, 0, 0, 0, 0};
struct fileTree *newdir, *currdir, *prevdir = NULL; struct fileTree *newdir, *currdir, *prevdir = NULL;
@ -515,13 +514,12 @@ findOrCreateFileTreeBrother(struct fileTree* *first,char* findname, int arch)
newdir->name = strdup(findname); newdir->name = strdup(findname);
return newdir; return newdir;
} }
return currdir; return currdir;
} }
struct fileTree* struct fileTree*
findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch) findOrCreateFileTreeEntry(struct fileTree* *first, char* findname, int arch)
{ {
struct fileTree *currdir,*prevdir=NULL; struct fileTree *currdir,*prevdir=NULL;
char *pstart,*pend; char *pstart,*pend;
@ -529,8 +527,8 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
int l; int l;
currdir = *first; currdir = *first;
pstart = findname+1; /* skip trailing slash */ pstart = &(findname[1]); /* skip trailing slash */
pend = findname+1; pend = &(findname[1]);
l = strlen(findname); l = strlen(findname);
while (pend-findname < l) { while (pend-findname < l) {
@ -542,7 +540,7 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
if (prevdir) { if (prevdir) {
currdir = findOrCreateFileTreeBrother(&prevdir->firstchild,f,arch); currdir = findOrCreateFileTreeBrother(&prevdir->firstchild,f,arch);
currdir->parent = prevdir; currdir->parent = prevdir;
} }
else else
currdir = findOrCreateFileTreeBrother(first,f,arch); currdir = findOrCreateFileTreeBrother(first,f,arch);
@ -552,8 +550,6 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
prevdir = currdir; prevdir = currdir;
pstart=pend+1; pstart=pend+1;
} }
// fprintf(stderr,"%s (%d) firstchild:%d\n",currdir->name,prevdir,currdir->firstchild);
return currdir; return currdir;
} }
@ -859,8 +855,8 @@ void getXMLPackageFiles(xmlNode *parent, uint **type, char ***path, int *count)
*path = NULL; *path = NULL;
return; return;
} }
*type = malloc(sizeof(int*) * (*count)); *type = malloc(sizeof(uint*) * (*count));
*path = malloc(sizeof(char*) * (*count)+1); *path = malloc(sizeof(char*) * (*count+1));
int i = 0; int i = 0;
for (xmlNode *entry=parent->children; entry; entry=entry->next) { for (xmlNode *entry=parent->children; entry; entry=entry->next) {
@ -1033,9 +1029,13 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
// Use repodata // Use repodata
//n = atoi((char*)findXMLPropertyByName(xmlDocGetRootElement(repodata->primary_doc), "packages")); //n = atoi((char*)findXMLPropertyByName(xmlDocGetRootElement(repodata->primary_doc), "packages"));
//xmlNode* packageNode = findXMLPackageByName(xmlDocGetRootElement(repodata->other_doc), "firefox"); //xmlNode* packageNode = findXMLPackageByName(xmlDocGetRootElement(repodata->other_doc), "firefox");
xmlNode *package_node; xmlNode *filelists_node = xmlDocGetRootElement(repodata->filelists_doc)->children;
for (package_node = xmlDocGetRootElement(repodata->primary_doc)->children; xmlNode *other_node = xmlDocGetRootElement(repodata->other_doc)->children;
package_node; package_node = package_node->next) { for (xmlNode *package_node = xmlDocGetRootElement(repodata->primary_doc)->children;
package_node;
package_node = package_node->next,
filelists_node = filelists_node->next,
other_node = other_node->next) {
if (!strcmp((char*)package_node->name, "package")) { if (!strcmp((char*)package_node->name, "package")) {
//print_element_names(package_node->children, 0); //print_element_names(package_node->children, 0);
newheadersourcelist = malloc(sizeof(struct headerSourceList)); newheadersourcelist = malloc(sizeof(struct headerSourceList));
@ -1099,7 +1099,7 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
(char*)findXMLAttributeByName(psize, "package")); (char*)findXMLAttributeByName(psize, "package"));
//hl->patch = headerGetStringArrayEntry(h, RPMTAG_PATCH, &count); //hl->patch = headerGetStringArrayEntry(h, RPMTAG_PATCH, &count);
// XML requires // Requires
getXMLPackageNFV(findXMLPropertyByName(format, "requires"), getXMLPackageNFV(findXMLPropertyByName(format, "requires"),
&requirename, &requireflags, &requireversion, &requirecount); &requirename, &requireflags, &requireversion, &requirecount);
newheadersourcelist->require = malloc(requirecount * newheadersourcelist->require = malloc(requirecount *
@ -1113,16 +1113,14 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
} }
newheadersourcelist->requirecount = requirecount; newheadersourcelist->requirecount = requirecount;
// XML Files // Files
xmlNode* filelistsNode = findXMLPackageByName(xmlDocGetRootElement( getXMLPackageFiles(filelists_node, &fileflags, &basenames, &filenamescount);
repodata->filelists_doc), newheadersourcelist->name);
getXMLPackageFiles(filelistsNode, &fileflags, &basenames, &filenamescount);
newheadersourcelist->source = basenames; newheadersourcelist->source = basenames;
newheadersourcelist->filenamecount = 0; newheadersourcelist->filenamecount = 0;
addNewToSourceHeaderList(newheadersourcelist, ct, altrepository);
// TODO: changelog // TODO: changelog
addNewToSourceHeaderList(newheadersourcelist, ct, altrepository);
} }
} }
cleanRepodata(repodata); cleanRepodata(repodata);
@ -1317,23 +1315,29 @@ int addNewToHeaderList(struct headerList **currheaderlist,
return 0; return 0;
} }
char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) { char* advanceXMLPackageNode(char* prevname, xmlNode **primary_node,
xmlNode **filelists_node) {
char* currname; char* currname;
// check if already currname > prevname // check if already currname > prevname
if (*package_node && !strcmp((char*)(*package_node)->name, "package")) { if (*primary_node && !strcmp((char*)(*primary_node)->name, "package")) {
currname = strdup((char*) currname = strdup((char*)
findXMLPropertyByName(*package_node, "name")->children->content); findXMLPropertyByName(*primary_node, "name")->children->content);
// package_node is already advanced // primary_node is already advanced
if (strcmp(prevname, currname) < 0) return currname; if (strcmp(prevname, currname) < 0) return currname;
free(currname); free(currname);
} }
// nope: go to next package // nope: go to next package
if (*package_node) *package_node = (*package_node)->next; if (*primary_node) {
while (*package_node && strcmp((char*)(*package_node)->name, "package")) *primary_node = (*primary_node)->next;
*package_node = (*package_node)->next; *filelists_node = (*filelists_node)->next;
if (*package_node) { }
while (*primary_node && strcmp((char*)(*primary_node)->name, "package")) {
*primary_node = (*primary_node)->next;
*filelists_node = (*filelists_node)->next;
}
if (*primary_node) {
currname = strdup((char*) currname = strdup((char*)
findXMLPropertyByName(*package_node, "name")->children->content); findXMLPropertyByName(*primary_node, "name")->children->content);
return currname; return currname;
} }
return NULL; return NULL;
@ -1345,7 +1349,7 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
int generateHeaderList(struct configTag* ct, int arch, int incremental) { int generateHeaderList(struct configTag* ct, int arch, int incremental) {
struct repoData* repodata[ALT_REPS_MAX + 1]; struct repoData* repodata[ALT_REPS_MAX + 1];
xmlNode *package_node[ALT_REPS_MAX + 1]; xmlNode *primary_node[ALT_REPS_MAX + 1], *filelists_node[ALT_REPS_MAX + 1];
char nextname[bufsize + 1]; char nextname[bufsize + 1];
char scanpath[bufsize + 1], *altscanpath[ALT_REPS_MAX]; char scanpath[bufsize + 1], *altscanpath[ALT_REPS_MAX];
@ -1417,7 +1421,8 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
if (repodata[0]) { if (repodata[0]) {
for (i = 0; i <= alt_reps_num; i++) { for (i = 0; i <= alt_reps_num; i++) {
package_node[i] = xmlDocGetRootElement(repodata[i]->primary_doc)->children; primary_node[i] = xmlDocGetRootElement(repodata[i]->primary_doc)->children;
filelists_node[i] = xmlDocGetRootElement(repodata[i]->filelists_doc)->children;
} }
} else { } else {
n = scansdir(scanpath, &namelist, rpmselector, scanrpmnamecmp); n = scansdir(scanpath, &namelist, rpmselector, scanrpmnamecmp);
@ -1448,7 +1453,8 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
char *newname = NULL; char *newname = NULL;
nextname[0] = '\0'; nextname[0] = '\0';
for (i = 0; i <= alt_reps_num; i++) { for (i = 0; i <= alt_reps_num; i++) {
newname = advanceXMLPackageNode(currname, &(package_node[i])); newname = advanceXMLPackageNode(currname, &(primary_node[i]),
&(filelists_node[i]));
if (newname) { if (newname) {
if (nextname[0] == '\0' || strcmp(nextname, newname) >= 0) { if (nextname[0] == '\0' || strcmp(nextname, newname) >= 0) {
strncpy(nextname, newname, bufsize); strncpy(nextname, newname, bufsize);
@ -1477,25 +1483,25 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
memset(newheaderlist, 0, sizeof(struct headerList)); memset(newheaderlist, 0, sizeof(struct headerList));
newheaderlist->name = strdup((char*) newheaderlist->name = strdup((char*)
findXMLPropertyByName(package_node[altidx], "name")->children->content); findXMLPropertyByName(primary_node[altidx], "name")->children->content);
xmlNode *version = findXMLPropertyByName(package_node[altidx], "version"); xmlNode *version = findXMLPropertyByName(primary_node[altidx], "version");
newheaderlist->epoch = atoi((char*)findXMLAttributeByName(version, "epoch")); newheaderlist->epoch = atoi((char*)findXMLAttributeByName(version, "epoch"));
newheaderlist->version = strdup((char*)findXMLAttributeByName(version, "ver")); newheaderlist->version = strdup((char*)findXMLAttributeByName(version, "ver"));
newheaderlist->release = strdup((char*)findXMLAttributeByName(version, "rel")); newheaderlist->release = strdup((char*)findXMLAttributeByName(version, "rel"));
newheaderlist->summary = strdup((char*) newheaderlist->summary = strdup((char*)
findXMLPropertyByName(package_node[altidx], "summary")->children->content); findXMLPropertyByName(primary_node[altidx], "summary")->children->content);
newheaderlist->arch = strdup((char*) newheaderlist->arch = strdup((char*)
findXMLPropertyByName(package_node[altidx], "arch")->children->content); findXMLPropertyByName(primary_node[altidx], "arch")->children->content);
if (findXMLPropertyByName(package_node[altidx], "description")->children) { if (findXMLPropertyByName(primary_node[altidx], "description")->children) {
newheaderlist->description = strdup((char*) newheaderlist->description = strdup((char*)
findXMLPropertyByName(package_node[altidx], "description")->children->content); findXMLPropertyByName(primary_node[altidx], "description")->children->content);
} }
xmlNode *format = findXMLPropertyByName(package_node[altidx], "format"); xmlNode *format = findXMLPropertyByName(primary_node[altidx], "format");
newheaderlist->group = strdup((char*) newheaderlist->group = strdup((char*)
findXMLPropertyByName(format, "group")->children->content); findXMLPropertyByName(format, "group")->children->content);
newheaderlist->sourcename = strdup((char*) newheaderlist->sourcename = strdup((char*)
findXMLPropertyByName(format, "sourcerpm")->children->content); findXMLPropertyByName(format, "sourcerpm")->children->content);
xmlNode *psize = findXMLPropertyByName(package_node[altidx], "size"); xmlNode *psize = findXMLPropertyByName(primary_node[altidx], "size");
newheaderlist->size = atoi( newheaderlist->size = atoi(
(char*)findXMLAttributeByName(psize, "package")); (char*)findXMLAttributeByName(psize, "package"));
@ -1552,6 +1558,7 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
newheaderlist->obsoleteversion = obsoleteversion; newheaderlist->obsoleteversion = obsoleteversion;
newheaderlist->obsoletecount = obsoletecount; newheaderlist->obsoletecount = obsoletecount;
newheaderlist->altrepository = altrepository; newheaderlist->altrepository = altrepository;
// Requires
newheaderlist->require = malloc(requirecount * sizeof(struct Require *)); newheaderlist->require = malloc(requirecount * sizeof(struct Require *));
for (j=0; j < requirecount; j++) { for (j=0; j < requirecount; j++) {
newheaderlist->require[j] = malloc(sizeof(struct Require)); newheaderlist->require[j] = malloc(sizeof(struct Require));
@ -1561,6 +1568,43 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
newheaderlist->require[j]->resolved = NULL; newheaderlist->require[j]->resolved = NULL;
} }
newheaderlist->requirecount = requirecount; newheaderlist->requirecount = requirecount;
// Files
getXMLPackageFiles(filelists_node[altidx], &fileflags, &basename,
&filenamecount);
newheaderlist->file = malloc(sizeof(struct fileTree*) * filenamecount);
newheaderlist->filenamecount = filenamecount;
for (j=0; j<filenamecount; j++) {
newheaderlist->file[j] = findOrCreateFileTreeEntry(&ct->filetree[arch], basename[j], arch);
free(basename[j]);
//newheaderlist->fileflags[j] = fileflags[j];
//newheaderlist->fileuser[j] = findOrCreateFileUserListEntry(&ct->fileuserlist[arch], NULL, arch);
//newheaderlist->filegroup[j] = findOrCreateFileGroupListEntry(&ct->filegrouplist[arch], NULL, arch);
if (newheaderlist->file[j]->numproviders == 0) {
newheaderlist->file[j]->numproviders++;
newheaderlist->file[j]->provider=malloc(sizeof(struct headerList*));
newheaderlist->file[j]->provider[0]=newheaderlist;
} else {
// check if the package is already providing it...
for (k = 0; k < newheaderlist->file[j]->numproviders; k++) {
if (newheaderlist->file[j]->provider[k] == newheaderlist)
break;
}
// if not add to the list of providers
if (k == newheaderlist->file[j]->numproviders) {
newheaderlist->file[j]->numproviders++;
newprovider=malloc(
sizeof(struct headerList*)*newheaderlist->file[j]->numproviders);
for (k = 0; k < newheaderlist->file[j]->numproviders-1; k++) {
newprovider[k]=newheaderlist->file[j]->provider[k];
}
newprovider[newheaderlist->file[j]->numproviders-1] = newheaderlist;
free(newheaderlist->file[j]->provider);
newheaderlist->file[j]->provider=newprovider;
}
}
}
free(basename);
free(fileflags);
} }
} else { } else {