headerlist,backend-sqlite3: fix files db generation when using repodata

src/DistroqueryAPI.cpp

@@ -43,7 +43,8 @@ json DistroqueryAPI::configToJsonRepositories() {
             { "description", ct->description },
             { "parents", parents }
         };
-        j.push_back(repository);
+        if (ct->repodata_url != NULL)
+           j.push_back(repository);
         ct = ct->next;
     }
     return j;
@@ -83,6 +84,11 @@ json DistroqueryAPI::getRepositoryPackages(string repository, int per_page, int
         return j;
     }
 
+    if (ct->repodata_url == NULL) {
+        j["error"] = "repository with tag '" + repository + "' is not supported by this API";
+        return j;
+    }
+
     auto db = openRepositoryDatabase(ct);
     if (!db) {
         j["error"] = "error opening database for repository " + repository;
@@ -313,6 +319,11 @@ json DistroqueryAPI::getProvidersForRequirement(string repository, string requir
         return j;
     }
 
+    if (ct->repodata_url == NULL) {
+        j["error"] = "repository with tag '" + repository + "' is not supported by this API";
+        return j;
+    }
+
     auto db = openRepositoryDatabase(ct);
     if (!db) {
         j["error"] = "error opening sources database for repository " + string(ct->tag);
@@ -341,6 +352,11 @@ json DistroqueryAPI::getPackageSourceDetails(string repository, string package)
         return j;
     }
 
+    if (ct->repodata_url == NULL) {
+        j["error"] = "repository with tag '" + repository + "' is not supported by this API";
+        return j;
+    }
+
     auto db = openRepositoryDatabase(ct);
     if (!db) {
         j["error"] = "error opening source database for repository " + repository;
@@ -418,6 +434,11 @@ json DistroqueryAPI::getPackageDetails(string repository, string package, string
         return j;
     }
 
+    if (ct->repodata_url == NULL) {
+        j["error"] = "repository with tag '" + repository + "' is not supported by this API";
+        return j;
+    }
+
     auto db = openRepositoryDatabase(ct, arch);
     if (!db) {
         j["error"] = "error opening database for repository " + repository + " and arch " + arch;
@@ -532,6 +553,11 @@ json DistroqueryAPI::getRepositoryProblems(string repository) {
         return j;
     }
 
+    if (ct->repodata_url == NULL) {
+        j["error"] = "repository with tag '" + repository + "' is not supported by this API";
+        return j;
+    }
+
     auto db = openRepositoryDatabase(ct);
     if (!db) {
         j["error"] = "error opening sources database for repository " + string(ct->tag);
@@ -540,9 +566,9 @@ json DistroqueryAPI::getRepositoryProblems(string repository) {
 
     attachCtDatabases(ct, db);
 
-    j["problems"] = json::array();
+    j["warnings"] = json::array();
 
-    // Check build requires
+    // Add warnings
     string sql = "SELECT * FROM warnings ORDER BY repository,arch,name";
     if (sqlite3_prepare_v2(db, sql.c_str(), sql.length(), &stmt, NULL) == SQLITE_OK) {
         while (sqlite3_step(stmt) == SQLITE_ROW) {
@@ -551,9 +577,46 @@ json DistroqueryAPI::getRepositoryProblems(string repository) {
             problem["arch"] = reinterpret_cast<const char*>(sqlite3_column_text(stmt,sqlite3_find_column_id(stmt, NULL, "arch")));
             problem["name"] = reinterpret_cast<const char*>(sqlite3_column_text(stmt,sqlite3_find_column_id(stmt, NULL, "name")));
             problem["text"] = reinterpret_cast<const char*>(sqlite3_column_text(stmt,sqlite3_find_column_id(stmt, NULL, "text")));
-            j["problems"].push_back(problem);
+            j["warnings"].push_back(problem);
         }
         sqlite3_finalize(stmt);
+    } else {
+        j["error"] = "error preparing query '" + sql + "': " + sqlite3_errmsg(db);
+        return j;
+    }
+
+    j["needrebuild"] = json::array();
+
+    // Add need rebuild
+    sql = "SELECT * FROM needrebuild,sources WHERE id_source=sources.id ORDER BY sources.name,name,arch";
+    if (sqlite3_prepare_v2(db, sql.c_str(), sql.length(), &stmt, NULL) == SQLITE_OK) {
+        string lastSource = "";
+        json torebuild = json::array();
+        while (sqlite3_step(stmt) == SQLITE_ROW) {
+            string source = reinterpret_cast<const char*>(sqlite3_column_text(stmt,sqlite3_find_column_id(stmt, "sources", "name")));
+            if (lastSource != "" && source != lastSource) {
+                json needrebuild = {};
+                needrebuild["source"] = lastSource;
+                needrebuild["torebuild"] = torebuild;
+                j["needrebuild"].push_back(needrebuild);
+                torebuild = json::array();
+            }
+            json rebuild = {};
+            rebuild["name"] = reinterpret_cast<const char*>(sqlite3_column_text(stmt,sqlite3_find_column_id(stmt, NULL, "name")));
+            rebuild["arch"] = reinterpret_cast<const char*>(sqlite3_column_text(stmt,sqlite3_find_column_id(stmt, NULL, "arch")));
+            torebuild.push_back(rebuild);
+            lastSource = source;
+        }
+        if (lastSource != "") {
+            json needrebuild = {};
+            needrebuild["source"] = lastSource;
+            needrebuild["torebuild"] = torebuild;
+            j["needrebuild"].push_back(needrebuild);
+        }
+        sqlite3_finalize(stmt);
+    } else {
+        j["error"] = "error preparing query '" + sql + "': " + sqlite3_errmsg(db);
+        return j;
     }
 
     sqlite3_close(db);

src/backend-sqlite3.c

@@ -181,10 +181,10 @@ int generateSQLite_files(struct configTag* ct, sqlite3 *db, int arch) {
          for (i = 0; i < currpackage->filenamecount; i++) {
             snprintf(sqlite3_query, PATH_MAX, "INSERT INTO packages_files_rel VALUES(NULL,%ld,%ld,%ld,%ld,%d,?,?,?);",
                currpackage->id,
-               currpackage->file[i]->id,
+               (currpackage->file)[i]->id, 0L, 0L,
-               currpackage->fileuser[i]->id,
+               //(currpackage->fileuser)[i]->id,
-               currpackage->filegroup[i]->id,
+               //(currpackage->filegroup)[i]->id,
-               currpackage->fileflags[i]);
+               (currpackage->fileflags)[i]);
             if (sqlite3_prepare_v2(db, sqlite3_query, -1, &stmt, NULL)) {
                fprintf(stderr, "ERROR: sqlite3_prepare_v2: %s (%s)\n", sqlite3_errmsg(db), sqlite3_query);
                return 1;
@@ -641,8 +641,15 @@ generateSQLite_sources(struct configTag *ct, sqlite3 *db) {
           generateSQLite_add_warnings(db, currsource->firstwarning, ct->repository[currsource->altrepository]->tag);
        }
 
-       if (currsource->firstrebuild) {
+       struct rebuildList *currfirstrebuild = currsource->firstrebuild;
-         generateSQLite_add_needrebuild(db, currsource->firstrebuild, currsource->id);
+       struct headerSourceList *oldheadersourcelist = currsource->old;
+       while (!currfirstrebuild && oldheadersourcelist) {
+          currfirstrebuild = oldheadersourcelist->firstrebuild;
+          oldheadersourcelist = oldheadersourcelist->old;
+       }
+
+       if (currfirstrebuild) {
+         generateSQLite_add_needrebuild(db, currfirstrebuild, currsource->id);
        }
 
        currsource = currsource->next;

src/buildtools.c

@@ -98,14 +98,14 @@ generateBuildInfo(struct configTag *configtag, int arch)
                i++;
           }
 
-          /* list filenames */
+          /* get specfile name */
-          for (i = 0; i < currheadersourcelist->filenamecount; i++) {
+          i = 0;
-               j = strlen((currheadersourcelist->basename)[i]) - 5;
+          while (currheadersourcelist->source[i] != NULL) {
+               j = strlen((currheadersourcelist->source)[i]) - 5;
                if (j > 0
-                   && !strcmp(&((currheadersourcelist->basename)[i])[j],
+                   && !strcmp(&((currheadersourcelist->source)[i])[j], ".spec")) {
-                              ".spec")) {
                     fprintf(fout, "SRPM_SPECFILE=%s\n",
-                            (currheadersourcelist->basename)[i]);
+                            (currheadersourcelist->source)[i]);
                     break;
                }
           }

src/headerlist.c

@@ -517,7 +517,7 @@ findOrCreateFileTreeEntry(struct fileTree* *first, char* findname, int arch)
     int l;
 
     currdir = *first;
-    pstart = &(findname[1]); /* skip trailing slash */
+    pstart = &(findname[1]); /* skip leading slash */
     pend = &(findname[1]);
     l = strlen(findname);
 
@@ -797,33 +797,36 @@ int XMLFlagToInt(char *flag) {
    return RPMSENSE_ANY;
 }
 
-void getXMLPackageFiles(xmlNode *parent, uint **type, char ***path, int *count) {
+void getXMLPackageFiles(xmlNode *parent, char ***basenames,
+      char ***usernames, char ***groupnames, int *filenamescount, int_16 **fileflags) {
 
-   *count = xmlChildElementCount(parent) - 1; // first child node is "version"
+   *filenamescount = 0;
-   if (*count == 0) {
-      *type = NULL;
-      *path = NULL;
-      return;
-   }
-   *type = malloc(sizeof(uint*) * (*count));
-   *path = malloc(sizeof(char*) * (*count+1));
 
-   int i = 0;
    for (xmlNode *entry=parent->children; entry; entry=entry->next) {
       if (entry->type == XML_ELEMENT_NODE && !strcmp((char*)entry->name, "file")) {
-         (*path)[i] = strdup((char*)entry->children->content);
+         (*filenamescount)++;
-         char* ftype = (char*)findXMLAttributeByName(entry, "type");
+      }
-         if (!ftype)
+   }
-            (*type)[i] = 0;
+
-         else if (!strcmp(ftype, "dir"))
+   *basenames = calloc(*filenamescount+1, sizeof(char*));
-            (*type)[i] = 1;
+   // FIXME: user, group and flags not currently available as XML file attributes
-         else if (!strcmp(ftype, "ghost"))
+   *usernames = calloc(*filenamescount+1, sizeof(char*));
-            (*type)[i] = 2;
+   *groupnames = calloc(*filenamescount+1, sizeof(char*));
-         i++;
+   *fileflags = calloc(*filenamescount+1, sizeof(char*));
+
+   int filescount = 0;
+   for (xmlNode *entry=parent->children; entry; entry=entry->next) {
+      if (entry->type == XML_ELEMENT_NODE && !strcmp((char*)entry->name, "file")) {
+         char* ftype = (char*)findXMLAttributeByName(entry, "type");
+         (*basenames)[filescount] = strdup((char*)entry->children->content);
+         if (ftype && !strcmp(ftype, "dir")) {
+            (*fileflags)[filescount] = S_IFDIR;
+         } else {
+            (*fileflags)[filescount] = S_IFREG;
+         }
+         filescount++;
       }
    }
-   // Null terminated list
-   (*path)[i] = NULL;
 }
 
 void getXMLPackageChangelog(xmlNode *parent,
@@ -1198,9 +1201,10 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
               newheadersourcelist->requirecount = requirecount;
 
               // Files
-              getXMLPackageFiles(filelists_node, &fileflags, &basenames, &filenamescount);
+              getXMLPackageFiles(filelists_node, &basenames, &usernames,
+                 &groupnames, &filenamescount, &fileflags);
+
               newheadersourcelist->source = basenames;
-              newheadersourcelist->filenamecount = 0;
 
               // Changelog
               if (mode & GENHEADER_CHANGELOG) {
@@ -1284,16 +1288,8 @@ int addToSourceHeaderList(struct configTag *ct, int mode, int altrepository) {
              getPackageRequires(h, &requirename, &requireflags, &requireversion,
                 &requirecount);
 
-             newheadersourcelist->dirindex = malloc(sizeof(int) * filenamescount);
+             newheadersourcelist->source =
-             if (!newheadersourcelist->dirindex) return 1;
-
-             memcpy(newheadersourcelist->dirindex, dirindexes,
-                sizeof(int) * filenamescount);
-             newheadersourcelist->dirname =
-                (char **) dupnargv(dirnames, dirnamescount);
-             newheadersourcelist->basename =
                 (char **) dupnargv(basenames, filenamescount);
-             newheadersourcelist->filenamecount = filenamescount;
              newheadersourcelist->require = malloc(requirecount *
                 sizeof(struct Require *));
              for (j=0; j < requirecount; j++) {
@@ -1718,16 +1714,21 @@ char* advanceXMLPackageNode(xmlNode **primary_node, xmlNode **filelists_node) {
            }
            newheaderlist->requirecount = requirecount;
            // Files
-           getXMLPackageFiles(filelists_node[altidx], &fileflags, &basename,
+           getXMLPackageFiles(filelists_node[altidx], &basename,
-              &filenamecount);
+              &fileusername, &filegroupname, &filenamecount, &fileflags);
            newheaderlist->file = malloc(sizeof(struct fileTree*) * filenamecount);
+           newheaderlist->fileuser = malloc(sizeof(struct fileUserList*) * filenamecount);
+           newheaderlist->filegroup = malloc(sizeof(struct fileGroupList*) * filenamecount);
+           newheaderlist->fileflags = malloc(sizeof(rpmFlags) * filenamecount);
            newheaderlist->filenamecount = filenamecount;
            for (j=0; j<filenamecount; j++) {
               newheaderlist->file[j] = findOrCreateFileTreeEntry(&ct->filetree[arch], basename[j], arch);
               free(basename[j]);
-              //newheaderlist->fileflags[j] = fileflags[j];
+              newheaderlist->fileflags[j] = fileflags[j];
-              //newheaderlist->fileuser[j] = findOrCreateFileUserListEntry(&ct->fileuserlist[arch], NULL, arch);
+              if (fileusername[j])
-              //newheaderlist->filegroup[j] = findOrCreateFileGroupListEntry(&ct->filegrouplist[arch], NULL, arch);
+                 newheaderlist->fileuser[j] = findOrCreateFileUserListEntry(&ct->fileuserlist[arch], fileusername[j], arch);
+              if (filegroupname[j])
+                 newheaderlist->filegroup[j] = findOrCreateFileGroupListEntry(&ct->filegrouplist[arch], filegroupname[j], arch);
               if (newheaderlist->file[j]->numproviders == 0) {
                  newheaderlist->file[j]->numproviders++;
                  newheaderlist->file[j]->provider=malloc(sizeof(struct headerList*));
@@ -1753,6 +1754,8 @@ char* advanceXMLPackageNode(xmlNode **primary_node, xmlNode **filelists_node) {
               }
            }
            free(basename);
+           free(fileusername);
+           free(filegroupname);
            free(fileflags);
         }
 
@@ -1898,7 +1901,7 @@ char* advanceXMLPackageNode(xmlNode **primary_node, xmlNode **filelists_node) {
               newheaderlist->fileuser =
                  malloc(sizeof(struct fileUserList *) * filenamecount);
               newheaderlist->filegroup =
-                 malloc(sizeof(char *) * filenamecount);
+                 malloc(sizeof(struct fileGroupList *) * filenamecount);
               for (j=0; j<filenamecount; j++) {
                  snprintf(filename,bufsize,"%s%s",dirname[dirindex[j]],basename[j]);
                  newheaderlist->file[j] = findOrCreateFileTreeEntry(&ct->filetree[arch], filename, arch);

src/include/headerlist.h

@@ -126,11 +126,7 @@ struct headerSourceList {
      char *arch;
      char *buildarchs;
      char *excludearch;
-     int *dirindex;
      long size;
-     char **dirname;
-     char **basename;
-     int filenamecount;
      long buildtime;
      int requirecount;
      struct Require **require;