openmamba/distromatic
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

repodata scan: added support for files provides; added -p option to enable parallel per-arch processing (default: disabled)

Browse Source
This commit is contained in:
Silvan Calarco 2021-02-16 14:44:40 +01:00
parent 1aa74444e2
commit f804316c02
2 changed files with 118 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
src/distromatic.c
View File

@ -154,11 +154,12 @@ static const char *helpmsg[] = {
" --changelogsince <mmddyy> print changelog for all packages since given date",
"",
"Options:",
" -t, --tag use repository identified by tag in config file",
" -a, --arch specify a target architecture (default: " DEFAULT_ARCH ")",
" -d, --debug display debug level messages",
" -c, --conf <file> specify configuration file (default: " DEFAULT_CONFIGFILE ")",
" -q, --quiet suppress all messages excluding warnings and errors",
" -d, --debug display debug level messages",
" -p, --parallel enable parallel execution of per-arch processing",
" -t, --tag use repository identified by tag in config file",
" -v, --version display version and exit",
"",
"Examples:",
@ -942,13 +943,14 @@ main(int argc, char *argv[])
struct rebuildList *currrebuild;
int i = 0,hasbuilds[ARCHS_MAX],ptharg[ARCHS_MAX];
int parallel = 0;
pthread_t pth[ARCHS_MAX];
char warning[8096];
time_t start_time, stop_time;
/* options args for 'getopt_long()' */
static const char *optstring = "+a:c:t:qhvd";
static const char *optstring = "+a:c:t:qhvdp";
static struct option longopts[] = {
{ "deps-table", no_argument, 0, 0 },
{ "data-tables", no_argument, 0, 0 },
@ -971,6 +973,7 @@ main(int argc, char *argv[])
{ "tag", required_argument, 0, 't' },
{ "version", no_argument, 0, 'v' },
{ "debug", no_argument, 0, 'd' },
{ "parallel", no_argument, 0, 'p' },
{ 0, 0, 0, 0 }
};
@ -1065,6 +1068,9 @@ main(int argc, char *argv[])
log_debug_set(1);
logmsg(LOG_DEBUG,"debug logging enabled");
break;
case 'p':
parallel = 1;
break;
default:
fprintf(stderr,
"Fatal error while parsing command line arguments\n");
@ -1154,22 +1160,35 @@ main(int argc, char *argv[])
}
warning[0]=0;
for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
snprintf(&warning[strlen(warning)],sizeof(warning)-strlen(warning)," %s", configtag->arch[i]);
if (parallel) {
for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
snprintf(&warning[strlen(warning)],sizeof(warning)-strlen(warning)," %s", configtag->arch[i]);
}
if (!quietmode)
fprintf(stdout, "Scanning binary packages for archs:%s...\n",warning);
else
logmsg(LOG_MARK, "Binary packages check for archs:%s", warning);
}
if (!quietmode)
fprintf(stdout, "Scanning binary packages for archs:%s...\n",warning);
else
logmsg(LOG_MARK, "Binary packages check for archs:%s", warning);
for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
ptharg[i]=i;
pthread_create(&pth[i],NULL,threadArchScan,&ptharg[i]);
if (!parallel) {
if (!quietmode)
fprintf(stdout, "Scanning binary packages for arch %s...\n",
configtag->arch[i]);
else
logmsg(LOG_MARK, "Binary packages check for arch %s",
configtag->arch[i]);
threadArchScan(&ptharg[i]);
}
else
pthread_create(&pth[i],NULL,threadArchScan,&ptharg[i]);
} // archs threads loop
for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
pthread_join(pth[i], NULL);
}
if (parallel)
for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
pthread_join(pth[i], NULL);
}
if (!passed_arch) { // can't do missing builds and ports check in single arch mode
if (!quietmode)

128
src/headerlist.c
View File

@ -452,8 +452,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx,
}
currprovided = idx[idxbot];
while (currprovided &&
((c=strcmp(currprovided->name,findname)) < 0)) {
while (currprovided && ((c=strcmp(currprovided->name,findname)) < 0)) {
prevprovided = currprovided;
currprovided = currprovided->next;
}
@ -482,7 +481,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx,
}
struct fileTree*
findOrCreateFileTreeBrother(struct fileTree* *first,char* findname, int arch)
findOrCreateFileTreeBrother(struct fileTree* *first, char* findname, int arch)
{
static long id[ARCHS_MAX] = { 0, 0, 0, 0, 0};
struct fileTree *newdir, *currdir, *prevdir = NULL;
@ -515,13 +514,12 @@ findOrCreateFileTreeBrother(struct fileTree* *first,char* findname, int arch)
newdir->name = strdup(findname);
return newdir;
}
return currdir;
}
struct fileTree*
findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
findOrCreateFileTreeEntry(struct fileTree* *first, char* findname, int arch)
{
struct fileTree *currdir,*prevdir=NULL;
char *pstart,*pend;
@ -529,8 +527,8 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
int l;
currdir = *first;
pstart = findname+1; /* skip trailing slash */
pend = findname+1;
pstart = &(findname[1]); /* skip trailing slash */
pend = &(findname[1]);
l = strlen(findname);
while (pend-findname < l) {
@ -542,7 +540,7 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
if (prevdir) {
currdir = findOrCreateFileTreeBrother(&prevdir->firstchild,f,arch);
currdir->parent = prevdir;
currdir->parent = prevdir;
}
else
currdir = findOrCreateFileTreeBrother(first,f,arch);
@ -552,8 +550,6 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
prevdir = currdir;
pstart=pend+1;
}
// fprintf(stderr,"%s (%d) firstchild:%d\n",currdir->name,prevdir,currdir->firstchild);
return currdir;
}
@ -859,8 +855,8 @@ void getXMLPackageFiles(xmlNode *parent, uint **type, char ***path, int *count)
*path = NULL;
return;
}
*type = malloc(sizeof(int*) * (*count));
*path = malloc(sizeof(char*) * (*count)+1);
*type = malloc(sizeof(uint*) * (*count));
*path = malloc(sizeof(char*) * (*count+1));
int i = 0;
for (xmlNode *entry=parent->children; entry; entry=entry->next) {
@ -1033,9 +1029,13 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
// Use repodata
//n = atoi((char*)findXMLPropertyByName(xmlDocGetRootElement(repodata->primary_doc), "packages"));
//xmlNode* packageNode = findXMLPackageByName(xmlDocGetRootElement(repodata->other_doc), "firefox");
xmlNode *package_node;
for (package_node = xmlDocGetRootElement(repodata->primary_doc)->children;
package_node; package_node = package_node->next) {
xmlNode *filelists_node = xmlDocGetRootElement(repodata->filelists_doc)->children;
xmlNode *other_node = xmlDocGetRootElement(repodata->other_doc)->children;
for (xmlNode *package_node = xmlDocGetRootElement(repodata->primary_doc)->children;
package_node;
package_node = package_node->next,
filelists_node = filelists_node->next,
other_node = other_node->next) {
if (!strcmp((char*)package_node->name, "package")) {
//print_element_names(package_node->children, 0);
newheadersourcelist = malloc(sizeof(struct headerSourceList));
@ -1099,7 +1099,7 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
(char*)findXMLAttributeByName(psize, "package"));
//hl->patch = headerGetStringArrayEntry(h, RPMTAG_PATCH, &count);
// XML requires
// Requires
getXMLPackageNFV(findXMLPropertyByName(format, "requires"),
&requirename, &requireflags, &requireversion, &requirecount);
newheadersourcelist->require = malloc(requirecount *
@ -1113,16 +1113,14 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
}
newheadersourcelist->requirecount = requirecount;
// XML Files
xmlNode* filelistsNode = findXMLPackageByName(xmlDocGetRootElement(
repodata->filelists_doc), newheadersourcelist->name);
getXMLPackageFiles(filelistsNode, &fileflags, &basenames, &filenamescount);
// Files
getXMLPackageFiles(filelists_node, &fileflags, &basenames, &filenamescount);
newheadersourcelist->source = basenames;
newheadersourcelist->filenamecount = 0;
addNewToSourceHeaderList(newheadersourcelist, ct, altrepository);
// TODO: changelog
addNewToSourceHeaderList(newheadersourcelist, ct, altrepository);
}
}
cleanRepodata(repodata);
@ -1317,23 +1315,29 @@ int addNewToHeaderList(struct headerList **currheaderlist,
return 0;
}
char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
char* advanceXMLPackageNode(char* prevname, xmlNode **primary_node,
xmlNode **filelists_node) {
char* currname;
// check if already currname > prevname
if (*package_node && !strcmp((char*)(*package_node)->name, "package")) {
if (*primary_node && !strcmp((char*)(*primary_node)->name, "package")) {
currname = strdup((char*)
findXMLPropertyByName(*package_node, "name")->children->content);
// package_node is already advanced
findXMLPropertyByName(*primary_node, "name")->children->content);
// primary_node is already advanced
if (strcmp(prevname, currname) < 0) return currname;
free(currname);
}
// nope: go to next package
if (*package_node) *package_node = (*package_node)->next;
while (*package_node && strcmp((char*)(*package_node)->name, "package"))
*package_node = (*package_node)->next;
if (*package_node) {
if (*primary_node) {
*primary_node = (*primary_node)->next;
*filelists_node = (*filelists_node)->next;
}
while (*primary_node && strcmp((char*)(*primary_node)->name, "package")) {
*primary_node = (*primary_node)->next;
*filelists_node = (*filelists_node)->next;
}
if (*primary_node) {
currname = strdup((char*)
findXMLPropertyByName(*package_node, "name")->children->content);
findXMLPropertyByName(*primary_node, "name")->children->content);
return currname;
}
return NULL;
@ -1345,7 +1349,7 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
int generateHeaderList(struct configTag* ct, int arch, int incremental) {
struct repoData* repodata[ALT_REPS_MAX + 1];
xmlNode *package_node[ALT_REPS_MAX + 1];
xmlNode *primary_node[ALT_REPS_MAX + 1], *filelists_node[ALT_REPS_MAX + 1];
char nextname[bufsize + 1];
char scanpath[bufsize + 1], *altscanpath[ALT_REPS_MAX];
@ -1417,7 +1421,8 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
if (repodata[0]) {
for (i = 0; i <= alt_reps_num; i++) {
package_node[i] = xmlDocGetRootElement(repodata[i]->primary_doc)->children;
primary_node[i] = xmlDocGetRootElement(repodata[i]->primary_doc)->children;
filelists_node[i] = xmlDocGetRootElement(repodata[i]->filelists_doc)->children;
}
} else {
n = scansdir(scanpath, &namelist, rpmselector, scanrpmnamecmp);
@ -1448,7 +1453,8 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
char *newname = NULL;
nextname[0] = '\0';
for (i = 0; i <= alt_reps_num; i++) {
newname = advanceXMLPackageNode(currname, &(package_node[i]));
newname = advanceXMLPackageNode(currname, &(primary_node[i]),
&(filelists_node[i]));
if (newname) {
if (nextname[0] == '\0' || strcmp(nextname, newname) >= 0) {
strncpy(nextname, newname, bufsize);
@ -1477,25 +1483,25 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
memset(newheaderlist, 0, sizeof(struct headerList));
newheaderlist->name = strdup((char*)
findXMLPropertyByName(package_node[altidx], "name")->children->content);
xmlNode *version = findXMLPropertyByName(package_node[altidx], "version");
findXMLPropertyByName(primary_node[altidx], "name")->children->content);
xmlNode *version = findXMLPropertyByName(primary_node[altidx], "version");
newheaderlist->epoch = atoi((char*)findXMLAttributeByName(version, "epoch"));
newheaderlist->version = strdup((char*)findXMLAttributeByName(version, "ver"));
newheaderlist->release = strdup((char*)findXMLAttributeByName(version, "rel"));
newheaderlist->summary = strdup((char*)
findXMLPropertyByName(package_node[altidx], "summary")->children->content);
findXMLPropertyByName(primary_node[altidx], "summary")->children->content);
newheaderlist->arch = strdup((char*)
findXMLPropertyByName(package_node[altidx], "arch")->children->content);
if (findXMLPropertyByName(package_node[altidx], "description")->children) {
findXMLPropertyByName(primary_node[altidx], "arch")->children->content);
if (findXMLPropertyByName(primary_node[altidx], "description")->children) {
newheaderlist->description = strdup((char*)
findXMLPropertyByName(package_node[altidx], "description")->children->content);
findXMLPropertyByName(primary_node[altidx], "description")->children->content);
}
xmlNode *format = findXMLPropertyByName(package_node[altidx], "format");
xmlNode *format = findXMLPropertyByName(primary_node[altidx], "format");
newheaderlist->group = strdup((char*)
findXMLPropertyByName(format, "group")->children->content);
newheaderlist->sourcename = strdup((char*)
findXMLPropertyByName(format, "sourcerpm")->children->content);
xmlNode *psize = findXMLPropertyByName(package_node[altidx], "size");
xmlNode *psize = findXMLPropertyByName(primary_node[altidx], "size");
newheaderlist->size = atoi(
(char*)findXMLAttributeByName(psize, "package"));
@ -1552,6 +1558,7 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
newheaderlist->obsoleteversion = obsoleteversion;
newheaderlist->obsoletecount = obsoletecount;
newheaderlist->altrepository = altrepository;
// Requires
newheaderlist->require = malloc(requirecount * sizeof(struct Require *));
for (j=0; j < requirecount; j++) {
newheaderlist->require[j] = malloc(sizeof(struct Require));
@ -1561,6 +1568,43 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
newheaderlist->require[j]->resolved = NULL;
}
newheaderlist->requirecount = requirecount;
// Files
getXMLPackageFiles(filelists_node[altidx], &fileflags, &basename,
&filenamecount);
newheaderlist->file = malloc(sizeof(struct fileTree*) * filenamecount);
newheaderlist->filenamecount = filenamecount;
for (j=0; j<filenamecount; j++) {
newheaderlist->file[j] = findOrCreateFileTreeEntry(&ct->filetree[arch], basename[j], arch);
free(basename[j]);
//newheaderlist->fileflags[j] = fileflags[j];
//newheaderlist->fileuser[j] = findOrCreateFileUserListEntry(&ct->fileuserlist[arch], NULL, arch);
//newheaderlist->filegroup[j] = findOrCreateFileGroupListEntry(&ct->filegrouplist[arch], NULL, arch);
if (newheaderlist->file[j]->numproviders == 0) {
newheaderlist->file[j]->numproviders++;
newheaderlist->file[j]->provider=malloc(sizeof(struct headerList*));
newheaderlist->file[j]->provider[0]=newheaderlist;
} else {
// check if the package is already providing it...
for (k = 0; k < newheaderlist->file[j]->numproviders; k++) {
if (newheaderlist->file[j]->provider[k] == newheaderlist)
break;
}
// if not add to the list of providers
if (k == newheaderlist->file[j]->numproviders) {
newheaderlist->file[j]->numproviders++;
newprovider=malloc(
sizeof(struct headerList*)*newheaderlist->file[j]->numproviders);
for (k = 0; k < newheaderlist->file[j]->numproviders-1; k++) {
newprovider[k]=newheaderlist->file[j]->provider[k];
}
newprovider[newheaderlist->file[j]->numproviders-1] = newheaderlist;
free(newheaderlist->file[j]->provider);
newheaderlist->file[j]->provider=newprovider;
}
}
}
free(basename);
free(fileflags);
}
} else {