cmake: use pkg-config for resolving dependencies

Changes:
 * Fix glib2 search path.
 * Add pkg-config support to CAP, GEOIP, GNUTLS, LUA, PortAudio (API
   19), zlib, kerberos.
 * Add pkg-config support to libnl3, libnl2 and libnl1 (but tested only
   with libnl3).

This makes it easier to do 32-bit builds on 64-bit hosts by just setting
`PKG_CONFIG_LIBDIR`. Due to how HINTS work, it is still fragile though:
missing 32-bit libraries will cause a fallback to 64-bit libraries.

A future patch could check for `<PREFIX>_FOUND` and remove the manual
`find_path` and `find_library` hackery since the paths are already
known.

Change-Id: Ieb4fb74695c96afb1a4c70168e84abb1fa4612c6
Reviewed-on: https://code.wireshark.org/review/4292
Reviewed-by: Anders Broman <a.broman58@gmail.com>

9 files changed, 106 insertions, 25 deletions

diff --git a/cmake/modules/FindCAP.cmake b/cmake/modules/FindCAP.cmake
index 09d3b86620..0ed69f671f 100644
--- a/cmake/modules/FindCAP.cmake
+++ b/cmake/modules/FindCAP.cmake
@@ -12,12 +12,19 @@ IF (CAP_INCLUDE_DIRS)
   SET(CAP_FIND_QUIETLY TRUE)
 ENDIF (CAP_INCLUDE_DIRS)
 
-FIND_PATH(CAP_INCLUDE_DIR sys/capability.h)
+find_package(PkgConfig)
+pkg_search_module(CAP libcap)
+
+FIND_PATH(CAP_INCLUDE_DIR sys/capability.h
+  HINTS ${CAP_INCLUDEDIR}
+)
 
 SET(CAP_NAMES cap)
-FIND_LIBRARY(CAP_LIBRARY NAMES ${CAP_NAMES} )
+FIND_LIBRARY(CAP_LIBRARY NAMES ${CAP_NAMES}
+  HINTS ${CAP_LIBDIR}
+)
 
-# handle the QUIETLY and REQUIRED arguments and set CAP_FOUND to TRUE if 
+# handle the QUIETLY and REQUIRED arguments and set CAP_FOUND to TRUE if
 # all listed variables are TRUE
 INCLUDE(FindPackageHandleStandardArgs)
 FIND_PACKAGE_HANDLE_STANDARD_ARGS(CAP DEFAULT_MSG CAP_LIBRARY CAP_INCLUDE_DIR)
diff --git a/cmake/modules/FindGEOIP.cmake b/cmake/modules/FindGEOIP.cmake
index c166eeec07..add3187eca 100644
--- a/cmake/modules/FindGEOIP.cmake
+++ b/cmake/modules/FindGEOIP.cmake
@@ -15,12 +15,23 @@ ENDIF (GEOIP_INCLUDE_DIRS)
 INCLUDE(FindWSWinLibs)
 FindWSWinLibs("GeoIP-.*" "GEOIP_HINTS")
 
-FIND_PATH(GEOIP_INCLUDE_DIR GeoIP.h HINTS "${GEOIP_HINTS}/include" )
+find_package(PkgConfig)
+pkg_search_module(GEOIP geoip)
+
+FIND_PATH(GEOIP_INCLUDE_DIR GeoIP.h
+  HINTS
+    "${GEOIP_INCLUDEDIR}"
+    "${GEOIP_HINTS}/include"
+)
 
 SET(GEOIP_NAMES GeoIP libGeoIP-1)
-FIND_LIBRARY(GEOIP_LIBRARY NAMES ${GEOIP_NAMES} HINTS "${GEOIP_HINTS}/lib" )
+FIND_LIBRARY(GEOIP_LIBRARY NAMES ${GEOIP_NAMES}
+  HINTS
+    "${GEOIP_LIBDIR}"
+    "${GEOIP_HINTS}/lib"
+  )
 
-# handle the QUIETLY and REQUIRED arguments and set GEOIP_FOUND to TRUE if 
+# handle the QUIETLY and REQUIRED arguments and set GEOIP_FOUND to TRUE if
 # all listed variables are TRUE
 INCLUDE(FindPackageHandleStandardArgs)
 FIND_PACKAGE_HANDLE_STANDARD_ARGS(GEOIP DEFAULT_MSG GEOIP_LIBRARY GEOIP_INCLUDE_DIR)
diff --git a/cmake/modules/FindGLIB2.cmake b/cmake/modules/FindGLIB2.cmake
index 8f8d3cbc99..a376fb42e4 100644
--- a/cmake/modules/FindGLIB2.cmake
+++ b/cmake/modules/FindGLIB2.cmake
@@ -40,7 +40,7 @@ find_path( GLIB2_MAIN_INCLUDE_DIR
 	NAMES
 		glib.h
 	HINTS
-		"${PC_GLIB2_INCLUDEDIR}"
+		"${GLIB2_INCLUDEDIR}"
 		"${GLIB2_HINTS}/include"
 	PATH_SUFFIXES
 		glib-2.0
@@ -58,7 +58,7 @@ find_library( GLIB2_LIBRARY
 		glib-2.0
 		libglib-2.0
 	HINTS
-		"${PC_GLIB2_LIBDIR}"
+		"${GLIB2_LIBDIR}"
 		"${GLIB2_HINTS}/lib"
 	PATHS
 		/opt/gnome/lib64
@@ -77,7 +77,7 @@ find_path( GLIB2_INTERNAL_INCLUDE_DIR
 	NAMES
 		glibconfig.h
 	HINTS
-		"${PC_GLIB2_INCLUDEDIR}"
+		"${GLIB2_INCLUDEDIR}"
 		"${glib2LibDir}"
 		${CMAKE_SYSTEM_LIBRARY_PATH}
 	PATH_SUFFIXES
diff --git a/cmake/modules/FindGNUTLS.cmake b/cmake/modules/FindGNUTLS.cmake
index f06835fc44..492fd8f41a 100644
--- a/cmake/modules/FindGNUTLS.cmake
+++ b/cmake/modules/FindGNUTLS.cmake
@@ -15,20 +15,32 @@ ENDIF (GNUTLS_INCLUDE_DIRS)
 INCLUDE(FindWSWinLibs)
 FindWSWinLibs("gnutls-.*" "GNUTLS_HINTS")
 
+find_package(PkgConfig)
+pkg_search_module(GNUTLS gnutls)
+
 FIND_PATH(GNUTLS_INCLUDE_DIR
-	NAMES
-	  gnutls.h
-	  gnutls/gnutls.h
-	PATH_SUFFIXES
-	  include
-	HINTS
-	  ${GNUTLS_HINTS}
+  NAMES
+    gnutls.h
+    gnutls/gnutls.h
+  PATH_SUFFIXES
+    include
+  HINTS
+    "${GNUTLS_INCLUDEDIR}"
+    "${GNUTLS_HINTS}"
 )
 
 SET(GNUTLS_NAMES gnutls libgnutls-28)
-FIND_LIBRARY(GNUTLS_LIBRARY NAMES ${GNUTLS_NAMES} libgmp-10 libgcc_s_sjlj-1 libffi-6 libhogweed-2-4 libnettle-4-6 libp11-kit-0 libtasn1-6 HINTS "${GNUTLS_HINTS}/bin" )
+FIND_LIBRARY(GNUTLS_LIBRARY
+  NAMES
+    ${GNUTLS_NAMES}
+    libgmp-10 libgcc_s_sjlj-1 libffi-6 libhogweed-2-4 libnettle-4-6
+    libp11-kit-0 libtasn1-6
+  HINTS
+    "${GNUTLS_LIBDIR}"
+    "${GNUTLS_HINTS}/bin"
+)
 
-# handle the QUIETLY and REQUIRED arguments and set GNUTLS_FOUND to TRUE if 
+# handle the QUIETLY and REQUIRED arguments and set GNUTLS_FOUND to TRUE if
 # all listed variables are TRUE
 INCLUDE(FindPackageHandleStandardArgs)
 FIND_PACKAGE_HANDLE_STANDARD_ARGS(GNUTLS DEFAULT_MSG GNUTLS_LIBRARY GNUTLS_INCLUDE_DIR)
diff --git a/cmake/modules/FindKERBEROS.cmake b/cmake/modules/FindKERBEROS.cmake
index 10d769d1e7..4d41441638 100644
--- a/cmake/modules/FindKERBEROS.cmake
+++ b/cmake/modules/FindKERBEROS.cmake
@@ -15,10 +15,21 @@ ENDIF (KERBEROS_INCLUDE_DIRS)
 INCLUDE(FindWSWinLibs)
 FindWSWinLibs("kfw-.*" "KERBOROS_HINTS")
 
-FIND_PATH(KERBEROS_INCLUDE_DIR krb5.h HINTS "${KERBOROS_HINTS}/include" )
+find_package(PkgConfig)
+pkg_search_module(KERBEROS krb5)
+
+FIND_PATH(KERBEROS_INCLUDE_DIR krb5.h
+  HINTS
+    "${KERBEROS_INCLUDEDIR}"
+    "${KERBOROS_HINTS}/include"
+)
 
 SET(KERBEROS_NAMES krb5 krb5_32)
-FIND_LIBRARY(KERBEROS_LIBRARY NAMES ${KERBEROS_NAMES} HINTS "${KERBOROS_HINTS}/lib" )
+FIND_LIBRARY(KERBEROS_LIBRARY NAMES ${KERBEROS_NAMES}
+  HINTS
+    "${KERBEROS_LIBDIR}"
+    "${KERBOROS_HINTS}/lib"
+)
 
 # handle the QUIETLY and REQUIRED arguments and set KERBEROS_FOUND to TRUE if 
 # all listed variables are TRUE
@@ -28,7 +39,7 @@ FIND_PACKAGE_HANDLE_STANDARD_ARGS(KERBEROS DEFAULT_MSG KERBEROS_LIBRARY KERBEROS
 # todo
 # add all kerberos libs
 # autodetect HAVE_HEIMDAL_KERBEROS
-# autodetect HAVE_MIT_KERBEROS
+# autodetect HAVE_MIT_KERBEROS (use pkg_search_module(mit-krb5)?)
 # autodetect(?) HAVE_KEYTYPE_ARCFOUR_56
 
 IF(KERBEROS_FOUND)
diff --git a/cmake/modules/FindLUA.cmake b/cmake/modules/FindLUA.cmake
index 7cbcae4a1d..18489a8d83 100644
--- a/cmake/modules/FindLUA.cmake
+++ b/cmake/modules/FindLUA.cmake
@@ -15,9 +15,13 @@
 INCLUDE(FindWSWinLibs)
 FindWSWinLibs("lua5*" "LUA_HINTS")
 
+find_package(PkgConfig)
+pkg_search_module(LUA lua)
+
 FIND_PATH(LUA_INCLUDE_DIR lua.h
   HINTS
-  $ENV{LUA_DIR}
+    "${LUA_INCLUDEDIR}"
+    "$ENV{LUA_DIR}"
   ${LUA_HINTS}
   PATH_SUFFIXES include/lua52 include/lua5.2 include/lua51 include/lua5.1 include/lua include
   PATHS
@@ -49,7 +53,8 @@ message("LUA INCLUDE SUFFIX: ${LUA_INC_SUFFIX}")
 FIND_LIBRARY(LUA_LIBRARY
   NAMES lua${LUA_INC_SUFFIX} lua52 lua5.2 lua51 lua5.1 lua
   HINTS
-  $ENV{LUA_DIR}
+    "${LUA_LIBDIR}"
+    "$ENV{LUA_DIR}"
   ${LUA_HINTS}
   PATH_SUFFIXES lib64 lib
   PATHS
diff --git a/cmake/modules/FindNL.cmake b/cmake/modules/FindNL.cmake
index dea96ba4b0..46ca03dbef 100644
--- a/cmake/modules/FindNL.cmake
+++ b/cmake/modules/FindNL.cmake
@@ -20,11 +20,19 @@ ELSE (NL_LIBRARIES AND NL_INCLUDE_DIRS )
       /usr
       /usr/local
   )
+
+  find_package(PkgConfig)
+  pkg_check_modules(NL3 libnl-3.0 libnl-genl-3.0 libnl-route-3.0)
+  pkg_search_module(NL2 libnl-2.0)
+
   FIND_PATH( NL_INCLUDE_DIR
     PATH_SUFFIXES
       include/libnl3
     NAMES
       netlink/version.h
+    HINTS
+      "${NL3_libnl-3.0_INCLUDEDIR}"
+      "${NL2_INCLUDEDIR}"
     PATHS
       $(SEARCHPATHS)
   )
@@ -35,6 +43,9 @@ ELSE (NL_LIBRARIES AND NL_INCLUDE_DIRS )
         nl-3 nl
       PATH_SUFFIXES
         lib64 lib
+      HINTS
+        "${NL3_libnl-3.0_LIBDIR}"
+        "${NL2_LIBDIR}"
       PATHS
         $(SEARCHPATHS)
     )
@@ -43,6 +54,9 @@ ELSE (NL_LIBRARIES AND NL_INCLUDE_DIRS )
         nl-genl-3 nl-genl
       PATH_SUFFIXES
         lib64 lib
+      HINTS
+        "${NL3_libnl-genl-3.0_LIBDIR}"
+        "${NL2_LIBDIR}"
       PATHS
         $(SEARCHPATHS)
     )
@@ -51,6 +65,9 @@ ELSE (NL_LIBRARIES AND NL_INCLUDE_DIRS )
         nl-route-3 nl-route
       PATH_SUFFIXES
         lib64 lib
+      HINTS
+        "${NL3_libnl-route-3.0_LIBDIR}"
+        "${NL2_LIBDIR}"
       PATHS
         $(SEARCHPATHS)
     )
@@ -71,9 +88,12 @@ ELSE (NL_LIBRARIES AND NL_INCLUDE_DIRS )
     ENDIF( NL_LIBRARY )
   ELSE( NL_INCLUDE_DIR )
     # NL version 1 ?
+    pkg_search_module(NL1 libnl-1)
     FIND_PATH( NL_INCLUDE_DIR
       NAMES
         netlink/netlink.h
+      HINTS
+        "${NL1_INCLUDEDIR}"
       PATHS
         $(SEARCHPATHS)
     )
@@ -82,6 +102,8 @@ ELSE (NL_LIBRARIES AND NL_INCLUDE_DIRS )
         nl
       PATH_SUFFIXES
         lib64 lib
+      HINTS
+        "${NL1_LIBDIR}"
       PATHS
         $(SEARCHPATHS)
     )
diff --git a/cmake/modules/FindPORTAUDIO.cmake b/cmake/modules/FindPORTAUDIO.cmake
index fd456358c0..d8a0d351d2 100644
--- a/cmake/modules/FindPORTAUDIO.cmake
+++ b/cmake/modules/FindPORTAUDIO.cmake
@@ -15,10 +15,18 @@ ENDIF (PORTAUDIO_INCLUDE_DIRS)
 INCLUDE(FindWSWinLibs)
 FindWSWinLibs("portaudio_.*" "PORTAUDIO_HINTS")
 
-FIND_PATH(PORTAUDIO_INCLUDE_DIR portaudio.h HINTS "${PORTAUDIO_HINTS}/include" )
+find_package(PkgConfig)
+pkg_search_module(PORTAUDIO portaudio-2.0)
+
+FIND_PATH(PORTAUDIO_INCLUDE_DIR portaudio.h
+  HINTS
+    "${PORTAUDIO_INCLUDEDIR}"
+    "${PORTAUDIO_HINTS}/include"
+)
 
 SET(PORTAUDIO_NAMES portaudio)
-FIND_LIBRARY(PORTAUDIO_LIBRARY NAMES ${PORTAUDIO_NAMES} )
+FIND_LIBRARY(PORTAUDIO_LIBRARY NAMES ${PORTAUDIO_NAMES}
+  HINTS "${PORTAUDIO_LIBDIR}")
 
 # handle the QUIETLY and REQUIRED arguments and set PORTAUDIO_FOUND to TRUE if
 # all listed variables are TRUE
diff --git a/cmake/modules/FindZLIB.cmake b/cmake/modules/FindZLIB.cmake
index 0faf7aa1b9..956046f075 100644
--- a/cmake/modules/FindZLIB.cmake
+++ b/cmake/modules/FindZLIB.cmake
@@ -35,10 +35,14 @@
 INCLUDE(FindWSWinLibs)
 FindWSWinLibs("zlib" "ZLIB_HINTS")
 
+find_package(PkgConfig)
+pkg_search_module(ZLIB zlib)
+
 FIND_PATH(ZLIB_INCLUDE_DIR
     NAMES
         zlib.h
     HINTS
+        "${ZLIB_INCLUDEDIR}"
         ${ZLIB_HINTS}/include
         ${ZLIB_HINTS}
     PATHS
@@ -50,6 +54,7 @@ FIND_LIBRARY(ZLIB_LIBRARY
     NAMES
         ${ZLIB_NAMES}
     HINTS
+        "${ZLIB_LIBDIR}"
         ${ZLIB_HINTS}/lib
         ${ZLIB_HINTS}
     PATHS