repodata scan: added support for files provides; added -p option to enable parallel per-arch processing (default: disabled)

src/distromatic.c

@@ -154,11 +154,12 @@ static const char *helpmsg[] = {
 " --changelogsince <mmddyy>  print changelog for all packages since given date",
 "",
 "Options:",
-" -t, --tag                  use repository identified by tag in config file",
 " -a, --arch                 specify a target architecture (default: " DEFAULT_ARCH ")",
+" -d, --debug                display debug level messages",
 " -c, --conf <file>          specify configuration file (default: " DEFAULT_CONFIGFILE ")",
 " -q, --quiet                suppress all messages excluding warnings and errors",
-" -d, --debug                display debug level messages",
+" -p, --parallel             enable parallel execution of per-arch processing",
+" -t, --tag                  use repository identified by tag in config file",
 " -v, --version              display version and exit",
 "",
 "Examples:",
@@ -942,13 +943,14 @@ main(int argc, char *argv[])
      struct rebuildList *currrebuild;
 
      int i = 0,hasbuilds[ARCHS_MAX],ptharg[ARCHS_MAX];
+     int parallel = 0;
      pthread_t pth[ARCHS_MAX];
      char warning[8096];
 
      time_t start_time, stop_time;
 
      /* options args for 'getopt_long()' */
-     static const char *optstring = "+a:c:t:qhvd";
+     static const char *optstring = "+a:c:t:qhvdp";
      static struct option longopts[] = {
           { "deps-table",     no_argument, 	 0,  0  },
           { "data-tables",    no_argument, 	 0,  0  },
@@ -971,6 +973,7 @@ main(int argc, char *argv[])
           { "tag",            required_argument, 0, 't' },
           { "version",        no_argument,       0, 'v' },
           { "debug",          no_argument,       0, 'd' },
+          { "parallel",       no_argument,       0, 'p' },
           { 0, 0, 0, 0 }
      };
 
@@ -1065,6 +1068,9 @@ main(int argc, char *argv[])
                log_debug_set(1);
                logmsg(LOG_DEBUG,"debug logging enabled");
                break;
+          case 'p':
+               parallel = 1;
+               break;
           default:
                fprintf(stderr,
                   "Fatal error while parsing command line arguments\n");
@@ -1154,6 +1160,7 @@ main(int argc, char *argv[])
           }
 
           warning[0]=0;
+          if (parallel) {
             for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
                snprintf(&warning[strlen(warning)],sizeof(warning)-strlen(warning)," %s", configtag->arch[i]);
             }
@@ -1161,12 +1168,24 @@ main(int argc, char *argv[])
                fprintf(stdout, "Scanning binary packages for archs:%s...\n",warning);
             else
                logmsg(LOG_MARK, "Binary packages check for archs:%s", warning);
+          }
 
           for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
              ptharg[i]=i;
+             if (!parallel) {
+                if (!quietmode)
+                   fprintf(stdout, "Scanning binary packages for arch %s...\n",
+                      configtag->arch[i]);
+                else
+                   logmsg(LOG_MARK, "Binary packages check for arch %s",
+                      configtag->arch[i]);
+                threadArchScan(&ptharg[i]);
+             }
+             else
                 pthread_create(&pth[i],NULL,threadArchScan,&ptharg[i]);
           } // archs threads loop
 
+          if (parallel)
              for (i = 0; i < ARCHS_MAX && configtag->arch[i]; i++) {
                 pthread_join(pth[i], NULL);
              }

src/headerlist.c

@@ -452,8 +452,7 @@ findOrCreateProvidedListEntry(struct providedList* *idx,
     }
 
     currprovided = idx[idxbot];
-    while (currprovided &&
+    while (currprovided && ((c=strcmp(currprovided->name,findname)) < 0)) {
-           ((c=strcmp(currprovided->name,findname)) < 0)) {
        prevprovided = currprovided;
        currprovided = currprovided->next;
     }
@@ -515,7 +514,6 @@ findOrCreateFileTreeBrother(struct fileTree* *first,char* findname, int arch)
        newdir->name = strdup(findname);
        return newdir;
     }
-
     return currdir;
 }
 
@@ -529,8 +527,8 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
     int l;
 
     currdir = *first;
-    pstart = findname+1; /* skip trailing slash */
+    pstart = &(findname[1]); /* skip trailing slash */
-    pend = findname+1;
+    pend = &(findname[1]);
     l = strlen(findname);
 
     while (pend-findname < l) {
@@ -552,8 +550,6 @@ findOrCreateFileTreeEntry(struct fileTree* *first,char* findname, int arch)
        prevdir = currdir;
        pstart=pend+1;
     }
-//       fprintf(stderr,"%s (%d) firstchild:%d\n",currdir->name,prevdir,currdir->firstchild);
-
     return currdir;
 }
 
@@ -859,8 +855,8 @@ void getXMLPackageFiles(xmlNode *parent, uint **type, char ***path, int *count)
       *path = NULL;
       return;
    }
-   *type = malloc(sizeof(int*) * (*count));
+   *type = malloc(sizeof(uint*) * (*count));
-   *path = malloc(sizeof(char*) * (*count)+1);
+   *path = malloc(sizeof(char*) * (*count+1));
 
    int i = 0;
    for (xmlNode *entry=parent->children; entry; entry=entry->next) {
@@ -1033,9 +1029,13 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
         // Use repodata
         //n = atoi((char*)findXMLPropertyByName(xmlDocGetRootElement(repodata->primary_doc), "packages"));
         //xmlNode* packageNode = findXMLPackageByName(xmlDocGetRootElement(repodata->other_doc), "firefox");
-        xmlNode *package_node;
+        xmlNode *filelists_node = xmlDocGetRootElement(repodata->filelists_doc)->children;
-        for (package_node = xmlDocGetRootElement(repodata->primary_doc)->children;
+        xmlNode *other_node = xmlDocGetRootElement(repodata->other_doc)->children;
-            package_node; package_node = package_node->next) {
+        for (xmlNode *package_node = xmlDocGetRootElement(repodata->primary_doc)->children;
+              package_node;
+              package_node = package_node->next,
+              filelists_node = filelists_node->next,
+              other_node = other_node->next) {
            if (!strcmp((char*)package_node->name, "package")) {
               //print_element_names(package_node->children, 0);
               newheadersourcelist = malloc(sizeof(struct headerSourceList));
@@ -1099,7 +1099,7 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
                  (char*)findXMLAttributeByName(psize, "package"));
               //hl->patch = headerGetStringArrayEntry(h, RPMTAG_PATCH, &count);
 
-              // XML requires
+              // Requires
               getXMLPackageNFV(findXMLPropertyByName(format, "requires"),
                  &requirename, &requireflags, &requireversion, &requirecount);
               newheadersourcelist->require = malloc(requirecount *
@@ -1113,16 +1113,14 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
               }
               newheadersourcelist->requirecount = requirecount;
 
-              // XML Files
+              // Files
-              xmlNode* filelistsNode = findXMLPackageByName(xmlDocGetRootElement(
+              getXMLPackageFiles(filelists_node, &fileflags, &basenames, &filenamescount);
-                 repodata->filelists_doc), newheadersourcelist->name);
-
-              getXMLPackageFiles(filelistsNode, &fileflags, &basenames, &filenamescount);
               newheadersourcelist->source = basenames;
               newheadersourcelist->filenamecount = 0;
 
-              addNewToSourceHeaderList(newheadersourcelist, ct, altrepository);
               // TODO: changelog
+
+              addNewToSourceHeaderList(newheadersourcelist, ct, altrepository);
            }
         }
         cleanRepodata(repodata);
@@ -1317,23 +1315,29 @@ int addNewToHeaderList(struct headerList **currheaderlist,
     return 0;
 }
 
-char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
+char* advanceXMLPackageNode(char* prevname, xmlNode **primary_node,
+      xmlNode **filelists_node) {
    char* currname;
    // check if already currname > prevname
-   if (*package_node && !strcmp((char*)(*package_node)->name, "package")) {
+   if (*primary_node && !strcmp((char*)(*primary_node)->name, "package")) {
       currname = strdup((char*)
-        findXMLPropertyByName(*package_node, "name")->children->content);
+        findXMLPropertyByName(*primary_node, "name")->children->content);
-      // package_node is already advanced
+      // primary_node is already advanced
       if (strcmp(prevname, currname) < 0) return currname;
       free(currname);
    }
    // nope: go to next package
-   if (*package_node) *package_node = (*package_node)->next;
+   if (*primary_node) {
-   while (*package_node && strcmp((char*)(*package_node)->name, "package"))
+      *primary_node = (*primary_node)->next;
-      *package_node = (*package_node)->next;
+      *filelists_node = (*filelists_node)->next;
-   if (*package_node) {
+   }
+   while (*primary_node && strcmp((char*)(*primary_node)->name, "package")) {
+      *primary_node = (*primary_node)->next;
+      *filelists_node = (*filelists_node)->next;
+   }
+   if (*primary_node) {
       currname = strdup((char*)
-         findXMLPropertyByName(*package_node, "name")->children->content);
+         findXMLPropertyByName(*primary_node, "name")->children->content);
       return currname;
    }
    return NULL;
@@ -1345,7 +1349,7 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
  int generateHeaderList(struct configTag* ct, int arch, int incremental) {
 
    struct repoData* repodata[ALT_REPS_MAX + 1];
-   xmlNode *package_node[ALT_REPS_MAX + 1];
+   xmlNode *primary_node[ALT_REPS_MAX + 1], *filelists_node[ALT_REPS_MAX + 1];
    char nextname[bufsize + 1];
 
    char scanpath[bufsize + 1], *altscanpath[ALT_REPS_MAX];
@@ -1417,7 +1421,8 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
 
    if (repodata[0]) {
      for (i = 0; i <= alt_reps_num; i++) {
-        package_node[i] = xmlDocGetRootElement(repodata[i]->primary_doc)->children;
+        primary_node[i] = xmlDocGetRootElement(repodata[i]->primary_doc)->children;
+        filelists_node[i] = xmlDocGetRootElement(repodata[i]->filelists_doc)->children;
      }
    } else {
      n = scansdir(scanpath, &namelist, rpmselector, scanrpmnamecmp);
@@ -1448,7 +1453,8 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
         char *newname = NULL;
         nextname[0] = '\0';
         for (i = 0; i <= alt_reps_num; i++) {
-           newname = advanceXMLPackageNode(currname, &(package_node[i]));
+           newname = advanceXMLPackageNode(currname, &(primary_node[i]),
+              &(filelists_node[i]));
            if (newname) {
               if (nextname[0] == '\0' || strcmp(nextname, newname) >= 0) {
                  strncpy(nextname, newname, bufsize);
@@ -1477,25 +1483,25 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
         memset(newheaderlist, 0, sizeof(struct headerList));
 
         newheaderlist->name = strdup((char*)
-           findXMLPropertyByName(package_node[altidx], "name")->children->content);
+           findXMLPropertyByName(primary_node[altidx], "name")->children->content);
-        xmlNode *version = findXMLPropertyByName(package_node[altidx], "version");
+        xmlNode *version = findXMLPropertyByName(primary_node[altidx], "version");
         newheaderlist->epoch = atoi((char*)findXMLAttributeByName(version, "epoch"));
         newheaderlist->version = strdup((char*)findXMLAttributeByName(version, "ver"));
         newheaderlist->release = strdup((char*)findXMLAttributeByName(version, "rel"));
         newheaderlist->summary = strdup((char*)
-           findXMLPropertyByName(package_node[altidx], "summary")->children->content);
+           findXMLPropertyByName(primary_node[altidx], "summary")->children->content);
         newheaderlist->arch = strdup((char*)
-           findXMLPropertyByName(package_node[altidx], "arch")->children->content);
+           findXMLPropertyByName(primary_node[altidx], "arch")->children->content);
-        if (findXMLPropertyByName(package_node[altidx], "description")->children) {
+        if (findXMLPropertyByName(primary_node[altidx], "description")->children) {
            newheaderlist->description = strdup((char*)
-              findXMLPropertyByName(package_node[altidx], "description")->children->content);
+              findXMLPropertyByName(primary_node[altidx], "description")->children->content);
         }
-        xmlNode *format = findXMLPropertyByName(package_node[altidx], "format");
+        xmlNode *format = findXMLPropertyByName(primary_node[altidx], "format");
         newheaderlist->group = strdup((char*)
            findXMLPropertyByName(format, "group")->children->content);
         newheaderlist->sourcename = strdup((char*)
            findXMLPropertyByName(format, "sourcerpm")->children->content);
-        xmlNode *psize = findXMLPropertyByName(package_node[altidx], "size");
+        xmlNode *psize = findXMLPropertyByName(primary_node[altidx], "size");
         newheaderlist->size = atoi(
            (char*)findXMLAttributeByName(psize, "package"));
 
@@ -1552,6 +1558,7 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
            newheaderlist->obsoleteversion = obsoleteversion;
            newheaderlist->obsoletecount = obsoletecount;
            newheaderlist->altrepository = altrepository;
+           // Requires
            newheaderlist->require = malloc(requirecount * sizeof(struct Require *));
            for (j=0; j < requirecount; j++) {
               newheaderlist->require[j] = malloc(sizeof(struct Require));
@@ -1561,6 +1568,43 @@ char* advanceXMLPackageNode(char* prevname, xmlNode **package_node) {
               newheaderlist->require[j]->resolved = NULL;
            }
            newheaderlist->requirecount = requirecount;
+           // Files
+           getXMLPackageFiles(filelists_node[altidx], &fileflags, &basename,
+              &filenamecount);
+           newheaderlist->file = malloc(sizeof(struct fileTree*) * filenamecount);
+           newheaderlist->filenamecount = filenamecount;
+           for (j=0; j<filenamecount; j++) {
+              newheaderlist->file[j] = findOrCreateFileTreeEntry(&ct->filetree[arch], basename[j], arch);
+              free(basename[j]);
+              //newheaderlist->fileflags[j] = fileflags[j];
+              //newheaderlist->fileuser[j] = findOrCreateFileUserListEntry(&ct->fileuserlist[arch], NULL, arch);
+              //newheaderlist->filegroup[j] = findOrCreateFileGroupListEntry(&ct->filegrouplist[arch], NULL, arch);
+              if (newheaderlist->file[j]->numproviders == 0) {
+                 newheaderlist->file[j]->numproviders++;
+                 newheaderlist->file[j]->provider=malloc(sizeof(struct headerList*));
+                 newheaderlist->file[j]->provider[0]=newheaderlist;
+              } else {
+                 // check if the package is already providing it...
+                 for (k = 0; k < newheaderlist->file[j]->numproviders; k++) {
+                    if (newheaderlist->file[j]->provider[k] == newheaderlist)
+                    break;
+                 }
+                 // if not add to the list of providers
+                 if (k == newheaderlist->file[j]->numproviders) {
+                    newheaderlist->file[j]->numproviders++;
+                    newprovider=malloc(
+                       sizeof(struct headerList*)*newheaderlist->file[j]->numproviders);
+                    for (k = 0; k < newheaderlist->file[j]->numproviders-1; k++) {
+                       newprovider[k]=newheaderlist->file[j]->provider[k];
+                    }
+                    newprovider[newheaderlist->file[j]->numproviders-1] = newheaderlist;
+                    free(newheaderlist->file[j]->provider);
+                    newheaderlist->file[j]->provider=newprovider;
+                 }
+              }
+           }
+           free(basename);
+           free(fileflags);
         }
 
       } else {