Merge pull request #1518 from KLayout/git-for-packages

Git for packages

build.sh

@@ -39,6 +39,7 @@ HAVE_QT=1
 HAVE_PNG=0
 HAVE_CURL=0
 HAVE_EXPAT=0
+HAVE_GIT2=1
 
 RUBYINCLUDE=""
 RUBYINCLUDE2=""
@@ -209,6 +210,9 @@ while [ "$*" != "" ]; do
   -libexpat)
     HAVE_EXPAT=1
     ;;
+  -nolibgit2)
+    HAVE_GIT2=0
+    ;;
   -qt5)
     echo "*** WARNING: -qt5 option is ignored - Qt version is auto-detected now."
     ;;
@@ -265,6 +269,7 @@ while [ "$*" != "" ]; do
     echo "  -libcurl              Use libcurl instead of QtNetwork (for Qt<4.7)"
     echo "  -libexpat             Use libexpat instead of QtXml"
     echo "  -libpng               Use libpng instead of Qt for PNG generation"
+    echo "  -nolibgit2            Do not include libgit2 for Git package support"
     echo ""
     echo "Environment Variables:"
     echo ""
@@ -495,6 +500,9 @@ fi
 if [ $HAVE_PNG != 0 ]; then
   echo "    Uses libpng for PNG generation"
 fi
+if [ $HAVE_GIT2 != 0 ]; then
+  echo "    Uses libgit2 for Git access"
+fi
 if [ "$RPATH" = "" ]; then
   RPATH="$BIN"
 fi
@@ -578,6 +586,7 @@ echo "      HAVE_64BIT_COORD=$HAVE_64BIT_COORD"
 echo "      HAVE_CURL=$HAVE_CURL"
 echo "      HAVE_PNG=$HAVE_PNG"
 echo "      HAVE_EXPAT=$HAVE_EXPAT"
+echo "      HAVE_GIT2=$HAVE_GIT2"
 echo "      RPATH=$RPATH"
 
 mkdir -p $BUILD
@@ -650,6 +659,7 @@ qmake_options=(
   HAVE_CURL="$HAVE_CURL"
   HAVE_EXPAT="$HAVE_EXPAT"
   HAVE_PNG="$HAVE_PNG"
+  HAVE_GIT2="$HAVE_GIT2"
   PREFIX="$BIN"
   RPATH="$RPATH"
   KLAYOUT_VERSION="$KLAYOUT_VERSION"

scripts/makedeb.sh

@@ -19,16 +19,16 @@ fi
 # TODO: derive this list automatically?
 case $target in
 ubuntu16)
-  depends="libqt4-designer (>= 4.8.6), libqt4-xml (>= 4.8.6), libqt4-sql (>= 4.8.6), libqt4-network (>= 4.8.6), libqtcore4 (>= 4.8.6), libqtgui4 (>= 4.8.6), zlib1g (>= 1.2.8), libruby2.3 (>= 2.3.1), python3 (>= 3.5.1), libpython3.5 (>= 3.5.1), libstdc++6 (>= 4.6.3), libc6 (>= 2.15)"
+  depends="libqt4-designer (>= 4.8.6), libqt4-xml (>= 4.8.6), libqt4-sql (>= 4.8.6), libqt4-network (>= 4.8.6), libqtcore4 (>= 4.8.6), libqtgui4 (>= 4.8.6), zlib1g (>= 1.2.8), libgit2-24 (>= 0.24.0), libruby2.3 (>= 2.3.1), python3 (>= 3.5.1), libpython3.5 (>= 3.5.1), libstdc++6 (>= 4.6.3), libc6 (>= 2.15)"
   ;;
 ubuntu18)
-  depends="libqt4-designer (>= 4.8.7), libqt4-xml (>= 4.8.7), libqt4-sql (>= 4.8.7), libqt4-network (>= 4.8.7), libqtcore4 (>= 4.8.7), libqtgui4 (>= 4.8.7), zlib1g (>= 1.2.11), libruby2.5 (>= 2.5.1), python3 (>= 3.6.5), libpython3.6 (>= 3.6.5), libstdc++6 (>= 8), libc6 (>= 2.27)"
+  depends="libqt4-designer (>= 4.8.7), libqt4-xml (>= 4.8.7), libqt4-sql (>= 4.8.7), libqt4-network (>= 4.8.7), libqtcore4 (>= 4.8.7), libqtgui4 (>= 4.8.7), zlib1g (>= 1.2.11), libgit2-26 (>= 0.26.0), libruby2.5 (>= 2.5.1), python3 (>= 3.6.5), libpython3.6 (>= 3.6.5), libstdc++6 (>= 8), libc6 (>= 2.27)"
   ;;
 ubuntu20)
-  depends="libqt5core5a (>= 5.12.8), libqt5designer5 (>= 5.12.8), libqt5gui5 (>= 5.12.8), libqt5multimedia5 (>= 5.12.8), libqt5multimediawidgets5 (>= 5.12.8), libqt5network5 (>= 5.12.8), libqt5opengl5 (>= 5.12.8), libqt5printsupport5 (>= 5.12.8), libqt5sql5 (>= 5.12.8), libqt5svg5 (>= 5.12.8), libqt5widgets5 (>= 5.12.8), libqt5xml5 (>= 5.12.8), libqt5xmlpatterns5 (>= 5.12.8), zlib1g (>= 1.2.11), libruby2.7 (>= 2.7.0), python3 (>= 3.8.2), libpython3.8 (>= 3.8.2), libstdc++6 (>=10), libc6 (>= 2.31)" 
+  depends="libqt5core5a (>= 5.12.8), libqt5designer5 (>= 5.12.8), libqt5gui5 (>= 5.12.8), libqt5multimedia5 (>= 5.12.8), libqt5multimediawidgets5 (>= 5.12.8), libqt5network5 (>= 5.12.8), libqt5opengl5 (>= 5.12.8), libqt5printsupport5 (>= 5.12.8), libqt5sql5 (>= 5.12.8), libqt5svg5 (>= 5.12.8), libqt5widgets5 (>= 5.12.8), libqt5xml5 (>= 5.12.8), libqt5xmlpatterns5 (>= 5.12.8), zlib1g (>= 1.2.11), libgit2-28 (>= 0.28.4), libruby2.7 (>= 2.7.0), python3 (>= 3.8.2), libpython3.8 (>= 3.8.2), libstdc++6 (>=10), libc6 (>= 2.31)" 
   ;;
 ubuntu22)
-  depends="libqt5core5a (>= 5.15.3), libqt5designer5 (>= 5.15.3), libqt5gui5 (>= 5.15.3), libqt5multimedia5 (>= 5.15.3), libqt5multimediawidgets5 (>= 5.15.3), libqt5network5 (>= 5.15.3), libqt5opengl5 (>= 5.15.3), libqt5printsupport5 (>= 5.15.3), libqt5sql5 (>= 5.15.3), libqt5svg5 (>= 5.15.3), libqt5widgets5 (>= 5.15.3), libqt5xml5 (>= 5.15.3), libqt5xmlpatterns5 (>= 5.15.3), zlib1g (>= 1.2.11), libruby3.0 (>= 3.0.2), python3 (>= 3.10.4), libpython3.10 (>= 3.10.4), libstdc++6 (>=12), libc6 (>= 2.35)" 
+  depends="libqt5core5a (>= 5.15.3), libqt5designer5 (>= 5.15.3), libqt5gui5 (>= 5.15.3), libqt5multimedia5 (>= 5.15.3), libqt5multimediawidgets5 (>= 5.15.3), libqt5network5 (>= 5.15.3), libqt5opengl5 (>= 5.15.3), libqt5printsupport5 (>= 5.15.3), libqt5sql5 (>= 5.15.3), libqt5svg5 (>= 5.15.3), libqt5widgets5 (>= 5.15.3), libqt5xml5 (>= 5.15.3), libqt5xmlpatterns5 (>= 5.15.3), zlib1g (>= 1.2.11), libgit2-1.1 (>= 1.1.0), libruby3.0 (>= 3.0.2), python3 (>= 3.10.4), libpython3.10 (>= 3.10.4), libstdc++6 (>=12), libc6 (>= 2.35)" 
   ;;
 *)
   echo "Unknown target '$target' (given as first argument)"

scripts/rpm-data/klayout.spec

@@ -53,6 +53,7 @@ Requires: qt5-qttools-devel >= 5.11.1
 Requires:	ruby >= 2.0.0
 Requires:	python3 >= 3.6.0
 Requires: qt-x11 >= 4.8.5
+Requires: libgit2 >= 0.26.8
 %define buildopt -j2
 %endif
 
@@ -62,29 +63,30 @@ Requires: libcurl >= 7.19.7
 Requires: ruby >= 1.8.7
 Requires: python >= 2.6.6
 Requires: qt-x11 >= 4.6.2
-%define buildopt -libcurl -j2
+%define buildopt -libcurl -j2 -nolibgit2
 %endif
 
 %if "%{target_system}" == "opensuse42_2"
 # OpenSuSE 42.2 requirements
 Requires:	ruby2.3 >= 2.3.1
 Requires:	python3 >= 3.4.6
 Requires: libqt4-x11 >= 4.8.6
-%define buildopt -j2
+%define buildopt -j2 -nolibgit2
 %endif
 
 %if "%{target_system}" == "opensuse42_3"
 # OpenSuSE 42.3 requirements
 Requires:	ruby2.3 >= 2.3.1
 Requires:	python3 >= 3.4.6
 Requires: libqt4-x11 >= 4.8.6
-%define buildopt -j2
+%define buildopt -j2 -nolibgit2
 %endif
 
 %if "%{target_system}" == "opensuse15"
 # OpenSuSE Leap 15 requirements
 Requires:	ruby >= 2.5
 Requires:	python3 >= 3.6
+Requires:	libgit2 >= 1.3.0
 Requires: libqt5-qtbase >= 5.15.2
 Requires: libQt5PrintSupport5 >= 5.15.2
 Requires: libQt5Designer5 >= 5.15.2

setup.py

@@ -422,15 +422,12 @@ def macros(self):
         macros = [
             ("HAVE_CURL", 1),
             ("HAVE_EXPAT", 1),
+            ("HAVE_PNG", 1),
             ("KLAYOUT_MAJOR_VERSION", self.major_version()),
             ("KLAYOUT_MINOR_VERSION", self.minor_version()),
             ("GSI_ALIAS_INSPECT", 1),
         ]
 
-        if platform.system() == "Darwin" and check_libpng():
-                macros += [("HAVE_PNG", 1)]
-        else:
-            macros += [("HAVE_PNG", 1)]
         return macros
 
     def minor_version(self):

src/db/db/db.pro

@@ -407,7 +407,7 @@ HEADERS = \
   dbShapeCollection.h \
   dbShapeCollectionUtils.h
 
-!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") {
+!equals(HAVE_QT, "0") {
 
   RESOURCES = \
     dbResources.qrc \

src/drc/drc/drc.pro

@@ -13,7 +13,7 @@ HEADERS = \
   drcCommon.h \
   drcForceLink.h \
 
-!equals(HAVE_QT, "0") || !equals(HAVE_PYTHON, "0") {
+!equals(HAVE_QT, "0") {
   RESOURCES = \
     drcResources.qrc
 }

src/klayout.pri

@@ -103,6 +103,15 @@ equals(HAVE_PTHREADS, "1") {
   DEFINES += HAVE_PTHREADS
 }
 
+equals(HAVE_GIT2, "1") {
+  !isEmpty(BITS_PATH) {
+    include($$BITS_PATH/git2/git2.pri)
+  } else {
+    LIBS += -lgit2
+  }
+  DEFINES += HAVE_GIT2
+}
+
 equals(HAVE_RUBY, "1") {
   !isEmpty(BITS_PATH) {
     include($$BITS_PATH/ruby/ruby.pri)

src/lay/lay/lay.pro

@@ -33,6 +33,7 @@ HEADERS = \
     layResourceHelpProvider.h \
     layRuntimeErrorForm.h \
     layReaderErrorForm.h \
+    laySaltParsedURL.h \
     laySearchReplaceConfigPage.h \
     laySearchReplaceDialog.h \
     laySearchReplacePropertiesWidgets.h \
@@ -144,6 +145,7 @@ SOURCES = \
     layResourceHelpProvider.cc \
     layRuntimeErrorForm.cc \
     layReaderErrorForm.cc \
+    laySaltParsedURL.cc \
     laySearchReplaceConfigPage.cc \
     laySearchReplaceDialog.cc \
     laySearchReplacePlugin.cc \

src/lay/lay/layApplication.cc

@@ -770,7 +770,7 @@ ApplicationBase::init_app ()
     size_t local_folders = (lay::get_appdata_path ().empty () ? 0 : 1);
 
     for (std::vector <std::string>::const_iterator p = m_klayout_path.begin (); p != m_klayout_path.end (); ++p) {
-      if (p - m_klayout_path.begin () < local_folders) {
+      if (size_t (p - m_klayout_path.begin ()) < local_folders) {
         mc->add_path (*p, tl::to_string (QObject::tr ("Local")), std::string (), false);
       } else if (m_klayout_path.size () == 1 + local_folders) {
         mc->add_path (*p, tl::to_string (QObject::tr ("Global")), std::string (), true);

src/lay/lay/layInit.cc

@@ -152,7 +152,7 @@ void init (const std::vector<std::string> &_paths)
         try {
           s_plugins.push_back (do_load_plugin (imp));
           modules.insert (*im);
-        } catch (tl::Exception (&ex)) {
+        } catch (tl::Exception &ex) {
           tl::error << ex.msg ();
         }
       }

src/lay/lay/laySalt.cc

@@ -21,11 +21,18 @@
 */
 
 #include "laySalt.h"
+#include "laySaltParsedURL.h"
+
 #include "tlString.h"
 #include "tlFileUtils.h"
 #include "tlLog.h"
 #include "tlInternational.h"
 #include "tlWebDAV.h"
+#include "tlEnv.h"
+#if defined(HAVE_GIT2)
+#  include "tlGit.h"
+#endif
+
 #include "lymMacro.h"
 
 #include <QFileInfo>
@@ -63,6 +70,13 @@ Salt::root ()
   return m_root;
 }
 
+bool
+Salt::download_package_information () const
+{
+  //  $KLAYOUT_ALWAYS_DOWNLOAD_PACKAGE_INFO
+  return tl::app_flag ("always-download-package-info") || m_root.sparse ();
+}
+
 Salt::flat_iterator
 Salt::begin_flat ()
 {
@@ -483,11 +497,27 @@ Salt::create_grain (const SaltGrain &templ, SaltGrain &target, double timeout, t
 
   } else if (! templ.url ().empty ()) {
 
-    if (templ.url ().find ("http:") == 0 || templ.url ().find ("https:") == 0) {
+    lay::SaltParsedURL purl (templ.url ());
+
+    if (purl.url ().find ("http:") == 0 || purl.url ().find ("https:") == 0) {
 
-      //  otherwise download from the URL
-      tl::info << QObject::tr ("Downloading package from '%1' to '%2' ..").arg (tl::to_qstring (templ.url ())).arg (tl::to_qstring (target.path ()));
-      res = tl::WebDAVObject::download (templ.url (), target.path (), timeout, callback);
+      //  otherwise download from the URL using Git or SVN
+
+      if (purl.protocol () == Git) {
+
+#if defined(HAVE_GIT2)
+        tl::info << QObject::tr ("Downloading package from '%1' to '%2' using Git protocol (ref='%3', subdir='%4') ..").arg (tl::to_qstring (purl.url ())).arg (tl::to_qstring (target.path ())).arg (tl::to_qstring (purl.branch ())).arg (tl::to_qstring (purl.subfolder ()));
+        res = tl::GitObject::download (purl.url (), target.path (), purl.subfolder (), purl.branch (), timeout, callback);
+#else
+        throw tl::Exception (tl::to_string (QObject::tr ("Unable to install package '%1' - git protocol not compiled in").arg (tl::to_qstring (target.name ()))));
+#endif
+
+      } else if (purl.protocol () == WebDAV || purl.protocol () == DefaultProtocol) {
+
+        tl::info << QObject::tr ("Downloading package from '%1' to '%2' using SVN/WebDAV protocol ..").arg (tl::to_qstring (purl.url ())).arg (tl::to_qstring (target.path ()));
+        res = tl::WebDAVObject::download (purl.url (), target.path (), timeout, callback);
+
+      }
 
     } else {
 

src/lay/lay/laySalt.h

@@ -199,6 +199,11 @@ Q_OBJECT
    */
   SaltGrains &root ();
 
+  /**
+   *  @brief Gets a value indicating whether the collection wants package information to be downloaded always
+   */
+  bool download_package_information () const;
+
 signals:
   /**
    *  @brief A signal triggered before one of the collections changed

src/lay/lay/laySaltController.cc

@@ -23,6 +23,7 @@
 #include "laySaltController.h"
 #include "laySaltManagerDialog.h"
 #include "laySaltDownloadManager.h"
+#include "laySaltParsedURL.h"
 #include "layConfig.h"
 #include "layMainWindow.h"
 #include "layQtTools.h"
@@ -142,7 +143,7 @@ SaltController::show_editor ()
     {
       //  while running the dialog, don't watch file events - that would interfere with
       //  the changes applied by the dialog itself.
-      lay::BusySection busy_section;  //  disable file watcher
+      tl::FileSystemWatcherDisabled disable_file_watcher;  //  disable file watcher
       mp_salt_dialog->exec ();
     }
 
@@ -156,7 +157,7 @@ SaltController::show_editor ()
 void
 SaltController::sync_file_watcher ()
 {
-  lay::BusySection busy_section;  //  disable file watcher
+  tl::FileSystemWatcherDisabled disable_file_watcher;  //  disable file watcher
 
   if (m_file_watcher) {
     m_file_watcher->clear ();
@@ -202,12 +203,19 @@ SaltController::install_packages (const std::vector<std::string> &packages, bool
       }
     }
 
-    if (n.find ("http:") == 0 || n.find ("https:") == 0 || n.find ("file:") == 0 || n[0] == '/' || n[0] == '\\') {
+    lay::SaltParsedURL purl (n);
+    const std::string &url = purl.url ();
+
+    if (url.find ("http:") == 0 || url.find ("https:") == 0 || url.find ("file:") == 0 || url[0] == '/' || url[0] == '\\') {
+
       //  its a URL
       manager.register_download (std::string (), std::string (), n, v);
+
     } else {
+
       //  its a plain name
       manager.register_download (n, std::string (), std::string (), v);
+
     }
 
   }
@@ -223,7 +231,7 @@ SaltController::install_packages (const std::vector<std::string> &packages, bool
   {
     //  while running the dialog, don't watch file events - that would interfere with
     //  the changes applied by the dialog itself.
-    lay::BusySection busy_section;  //  disable file watcher
+    tl::FileSystemWatcherDisabled disable_file_watcher;  //  disable file watcher
     result = manager.execute (0, m_salt);
   }