Merge pull request #278 from probonopd/TheAssassin/fix-274

Improve excludelist generation (fixes #274)

tools/excludelist.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-
-set -e
-
-# Download excludelist
-blacklisted=($(wget --quiet https://raw.githubusercontent.com/probonopd/AppImages/master/excludelist -O - | sort | uniq | grep -v "^#.*" | grep "[^-\s]"))
-
-# Create array
-for item in ${blacklisted[@]:0:${#blacklisted[@]}-1}; do
-  echo -ne '\\"'$item'\\" << '
-done
-echo -ne '\\"'${blacklisted[-1]}'\\"'

tools/generate-excludelist.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -e
+
+# download excludelist
+blacklisted=($(wget --quiet https://raw.githubusercontent.com/probonopd/AppImages/master/excludelist -O - | sort | uniq | grep -v "^#.*" | grep "[^-\s]"))
+
+# sanity check
+if [ "$blacklisted" == "" ]; then
+    exit 1;
+fi
+
+filename=$(readlink -f $(dirname "$0"))/linuxdeployqt/excludelist.h
+
+# overwrite existing source file
+cat > "$filename" <<EOF
+/*
+ * List of libraries to exclude for different reasons.
+ *
+ * Automatically generated from
+ * https://raw.githubusercontent.com/probonopd/AppImages/master/excludelist
+ *
+ * This file shall be committed by the developers occassionally,
+ * otherwise systems without access to the internet won't be able to build
+ * fully working versions of linuxdeployqt.
+ *
+ * See https://github.com/probonopd/linuxdeployqt/issues/274 for more
+ * information.
+ */
+
+#include <QStringList>
+
+static const QStringList generatedExcludelist = {
+EOF
+
+# Create array
+for item in ${blacklisted[@]:0:${#blacklisted[@]}-1}; do
+    echo -e '    "'"$item"'",' >> "$filename"
+done
+echo -e '    "'"${blacklisted[-1]}"'"' >> "$filename"
+
+echo "};" >> "$filename"

tools/linuxdeployqt/CMakeLists.txt

@@ -1,5 +1,8 @@
 set(CMAKE_AUTOMOC ON)
 
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+
 # expose version data as compiler definition
 add_definitions("-DLINUXDEPLOYQT_VERSION=\"${GIT_TAG_NAME}\"")
 add_definitions("-DLINUXDEPLOYQT_GIT_COMMIT=\"${GIT_COMMIT}\"")
@@ -9,14 +12,15 @@ add_definitions("-DBUILD_NUMBER=\"${BUILD_NUMBER}\"")
 find_package(Qt5 REQUIRED COMPONENTS Core)
 
 # update excludelist
+message(STATUS "Updating excludelist...")
 execute_process(
-    COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/../excludelist.sh
+    COMMAND bash ${CMAKE_CURRENT_SOURCE_DIR}/../generate-excludelist.sh
     OUTPUT_VARIABLE EXCLUDELIST
     TIMEOUT 10
     RESULT_VARIABLE EXCLUDELIST_RESULT
 )
 if(NOT EXCLUDELIST_RESULT EQUAL 0)
-    message(FATAL_ERROR "Failed to fetch and generate excludelist")
+    message(WARNING "Updating excludelist failed, using outdated copy")
 endif()
 mark_as_advanced(EXCLUDELIST EXCLUDELIST_RESULT)
 

tools/linuxdeployqt/excludelist.h

@@ -0,0 +1,69 @@
+/*
+ * List of libraries to exclude for different reasons.
+ *
+ * Automatically generated from
+ * https://raw.githubusercontent.com/probonopd/AppImages/master/excludelist
+ *
+ * This file shall be committed by the developers occassionally,
+ * otherwise systems without access to the internet won't be able to build
+ * fully working versions of linuxdeployqt.
+ *
+ * See https://github.com/probonopd/linuxdeployqt/issues/274 for more
+ * information.
+ */
+
+#include <QStringList>
+
+static const QStringList generatedExcludelist = {
+    "ld-linux.so.2",
+    "ld-linux-x86-64.so.2",
+    "libanl.so.1",
+    "libasound.so.2",
+    "libBrokenLocale.so.1",
+    "libcidn.so.1",
+    "libcom_err.so.2",
+    "libcrypt.so.1",
+    "libc.so.6",
+    "libdl.so.2",
+    "libdrm.so.2",
+    "libexpat.so.1",
+    "libfontconfig.so.1",
+    "libfreetype.so.6",
+    "libgcc_s.so.1",
+    "libgdk_pixbuf-2.0.so.0",
+    "libgio-2.0.so.0",
+    "libglib-2.0.so.0",
+    "libGL.so.1",
+    "libgobject-2.0.so.0",
+    "libgpg-error.so.0",
+    "libharfbuzz.so.0",
+    "libICE.so.6",
+    "libjack.so.0",
+    "libkeyutils.so.1",
+    "libm.so.6",
+    "libmvec.so.1",
+    "libnsl.so.1",
+    "libnss_compat.so.2",
+    "libnss_db.so.2",
+    "libnss_dns.so.2",
+    "libnss_files.so.2",
+    "libnss_hesiod.so.2",
+    "libnss_nisplus.so.2",
+    "libnss_nis.so.2",
+    "libp11-kit.so.0",
+    "libpango-1.0.so.0",
+    "libpangocairo-1.0.so.0",
+    "libpangoft2-1.0.so.0",
+    "libpthread.so.0",
+    "libresolv.so.2",
+    "librt.so.1",
+    "libSM.so.6",
+    "libstdc++.so.6",
+    "libthread_db.so.1",
+    "libusb-1.0.so.0",
+    "libutil.so.1",
+    "libuuid.so.1",
+    "libX11.so.6",
+    "libxcb.so.1",
+    "libz.so.1"
+};

tools/linuxdeployqt/linuxdeployqt.pro

@@ -38,18 +38,12 @@ DEFINES += LINUXDEPLOYQT_VERSION="'\"$(shell cd $$PWD && git describe --tags $(s
 contains(DEFINES, EXCLUDELIST.*) {
     message("EXCLUDELIST specified, to use the most recent exclude list, please run qmake without EXCLUDELIST definition and with internet.")
 } else {
-    message("Creating exclude list.")
+    message("Updating exclude list...")
 
     # check whether command _would_ run successfully
     EXCLUDELIST_GENERATION_WORKS = FALSE
-    system($$_PRO_FILE_PWD_/../excludelist.sh): EXCLUDELIST_GENERATION_WORKS = TRUE
+    system($$_PRO_FILE_PWD_/../generate-excludelist.sh): EXCLUDELIST_GENERATION_WORKS = TRUE
     isEqual(EXCLUDELIST_GENERATION_WORKS, FALSE) {
-        error("Generating excludelist failed")
+        warning("Updating excludelist failed, using outdated copy")
     }
-
-    EXCLUDELIST = $$system($$_PRO_FILE_PWD_/../excludelist.sh)
-    isEmpty(EXCLUDELIST) {
-        error("Generated excludelist is empty")
-    }
-    DEFINES += EXCLUDELIST=\""$$EXCLUDELIST"\"
 }

tools/linuxdeployqt/main.cpp

@@ -34,6 +34,7 @@
 #include <QSettings>
 #include <QDirIterator>
 #include <sstream>
+#include "excludelist.h"
 
 int main(int argc, char **argv)
 {
@@ -60,6 +61,10 @@ int main(int argc, char **argv)
             // can just exit normally, version has been printed above
             return 0;
         }
+        if (argument == QByteArray("-show-exclude-libs")) {
+            qInfo() << generatedExcludelist;
+            return 0;
+        }
     }
 
     if (argc < 2 || (firstArgument.startsWith("-"))) {
@@ -426,9 +431,6 @@ int main(int argc, char **argv)
             LogDebug() << "Argument found:" << argument;
             int index = argument.indexOf("=");
             excludeLibs = QString(argument.mid(index + 1)).split(",");
-        } else if (argument == QByteArray("-show-exclude-libs")) {
-            qInfo() << EXCLUDELIST;
-            return 0;
         } else if (argument.startsWith("--")) {
             LogError() << "Error: arguments must not start with --, only -:" << argument << "\n";
             return 1;

tools/linuxdeployqt/shared.cpp

@@ -44,6 +44,7 @@
 #include <QRegularExpression>
 #include <QStandardPaths>
 #include "shared.h"
+#include "excludelist.h"
 
 QString appBinaryPath;
 bool runStripEnabled = true;
@@ -465,20 +466,13 @@ LibraryInfo parseLddLibraryLine(const QString &line, const QString &appDirPath,
         This is more suitable for bundling in a way that is portable between different distributions and target systems.
         Along the way, this also takes care of non-Qt libraries.
 
-        The excludelist can be updated by running
-        #/bin/bash
-        blacklisted=$(wget https://raw.githubusercontent.com/probonopd/AppImages/master/excludelist -O - | sort | uniq | grep -v "^#.*" | grep "[^-\s]")
-        for item in $blacklisted; do
-          echo -ne '"'$item'" << '
-        done
+        The excludelist can be updated by running the bundled script generate-excludelist.sh
         */
 
-        QStringList excludelist;
-        #ifndef EXCLUDELIST
-            #error "EXCLUDELIST not defined! Please have your build system download run excludelist.sh and add -DEXCLUDE_LIST=<result>"
-        #else
-            excludelist << EXCLUDELIST;
-        #endif
+        // copy generated excludelist
+        QStringList excludelist = generatedExcludelist;
+
+        // append exclude libs
         excludelist += excludeLibs;
 
         LogDebug() << "excludelist:" << excludelist;