From d76ca9a1d91fcc090fe2e49ceab7bf09f1cda782 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 21 Nov 2017 14:17:33 +0100 Subject: [PATCH 1/7] fixed migration settings.yml --- conans/__init__.py | 2 +- conans/client/migrations.py | 18 ++---------------- 2 files changed, 3 insertions(+), 17 deletions(-) diff --git a/conans/__init__.py b/conans/__init__.py index d74151f3611..926ba0a0b58 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -16,4 +16,4 @@ COMPLEX_SEARCH_CAPABILITY = "complex_search" SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] -__version__ = '0.29.0' +__version__ = '0.29.1' diff --git a/conans/client/migrations.py b/conans/client/migrations.py index cbb23278a20..4a2ad539436 100644 --- a/conans/client/migrations.py +++ b/conans/client/migrations.py @@ -2,8 +2,7 @@ import shutil from conans.client.client_cache import CONAN_CONF, PROFILES_FOLDER -from conans.errors import ConanException -from conans.migrations import Migrator, CONAN_VERSION +from conans.migrations import Migrator from conans.paths import EXPORT_SOURCES_DIR_OLD from conans.util.files import load, save from conans.model.version import Version @@ -41,8 +40,7 @@ def _make_migrations(self, old_version): # VERSION 0.1 if old_version is None: return - - if old_version < Version("0.25"): + if old_version < Version("0.29"): old_settings = """ os: Windows: @@ -79,18 +77,6 @@ def _make_migrations(self, old_version): """ self._update_settings_yml(old_settings) - if old_version < Version("0.20"): - conf_path = os.path.join(self.client_cache.conan_folder, CONAN_CONF) - if conf_path: - backup_path = conf_path + ".backup" - save(backup_path, load(conf_path)) - os.unlink(conf_path) - os.unlink(os.path.join(self.client_cache.conan_folder, CONAN_VERSION)) - self.out.warn("*" * 40) - self.out.warn("Migration: Your Conan version was too old.") - self.out.warn("Your old conan.conf file has been backup'd to: %s" % backup_path) - self.out.warn("*" * 40) - if old_version < Version("0.25"): from conans.paths import DEFAULT_PROFILE_NAME default_profile_path = os.path.join(self.client_cache.conan_folder, PROFILES_FOLDER, From 49467d4ae917cddb9a78e6f914229dfc047d88a0 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 22 Nov 2017 13:28:22 +0100 Subject: [PATCH 2/7] locks migration (#2059) --- conans/client/migrations.py | 50 ++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/conans/client/migrations.py b/conans/client/migrations.py index 4a2ad539436..d13dc6f2661 100644 --- a/conans/client/migrations.py +++ b/conans/client/migrations.py @@ -4,8 +4,9 @@ from conans.client.client_cache import CONAN_CONF, PROFILES_FOLDER from conans.migrations import Migrator from conans.paths import EXPORT_SOURCES_DIR_OLD -from conans.util.files import load, save +from conans.util.files import load, save, list_folder_subdirs from conans.model.version import Version +from conans.errors import ConanException class ClientMigrator(Migrator): @@ -41,6 +42,7 @@ def _make_migrations(self, old_version): if old_version is None: return if old_version < Version("0.29"): + _migrate_lock_files(self.client_cache, self.out) old_settings = """ os: Windows: @@ -49,17 +51,23 @@ def _make_migrations(self, old_version): Android: api_level: ANY iOS: - version: ["7.0", "7.1", "8.0", "8.1", "8.2", "8.3", "9.0", "9.1", "9.2", "9.3", "10.0", "10.1", "10.2", "10.3"] + version: ["7.0", "7.1", "8.0", "8.1", "8.2", "8.3", "9.0", "9.1", "9.2", "9.3", "10.0", "10.1", "10.2", "10.3", "11.0"] + watchOS: + version: ["4.0"] + tvOS: + version: ["11.0"] FreeBSD: SunOS: -arch: [x86, x86_64, ppc64le, ppc64, armv6, armv7, armv7hf, armv8, sparc, sparcv9, mips, mips64] + Arduino: + board: ANY +arch: [x86, x86_64, ppc64le, ppc64, armv6, armv7, armv7hf, armv8, sparc, sparcv9, mips, mips64, avr, armv7s, armv7k] compiler: sun-cc: - version: ["5.10", "5.11", "5.12", "5.13", "5.14"] - threads: [None, posix] - libcxx: [libCstd, libstdcxx, libstlport, libstdc++] + version: ["5.10", "5.11", "5.12", "5.13", "5.14"] + threads: [None, posix] + libcxx: [libCstd, libstdcxx, libstlport, libstdc++] gcc: - version: ["4.1", "4.4", "4.5", "4.6", "4.7", "4.8", "4.9", "5.1", "5.2", "5.3", "5.4", "6.1", "6.2", "6.3", "7.1"] + version: ["4.1", "4.4", "4.5", "4.6", "4.7", "4.8", "4.9", "5.1", "5.2", "5.3", "5.4", "6.1", "6.2", "6.3", "6.4", "7.1", "7.2"] libcxx: [libstdc++, libstdc++11] threads: [None, posix, win32] # Windows MinGW exception: [None, dwarf2, sjlj, seh] # Windows MinGW @@ -67,10 +75,10 @@ def _make_migrations(self, old_version): runtime: [MD, MT, MTd, MDd] version: ["8", "9", "10", "11", "12", "14", "15"] clang: - version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0"] + version: ["3.3", "3.4", "3.5", "3.6", "3.7", "3.8", "3.9", "4.0", "5.0"] libcxx: [libstdc++, libstdc++11, libc++] apple-clang: - version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1"] + version: ["5.0", "5.1", "6.0", "6.1", "7.0", "7.3", "8.0", "8.1", "9.0"] libcxx: [libstdc++, libc++] build_type: [None, Debug, Release] @@ -92,6 +100,30 @@ def _make_migrations(self, old_version): migrate_c_src_export_source(self.client_cache, self.out) +def _migrate_lock_files(client_cache, out): + out.warn("Migration: Removing old lock files") + base_dir = client_cache.store + pkgs = list_folder_subdirs(base_dir, 4) + for pkg in pkgs: + out.info("Removing locks for %s" % pkg) + try: + count = os.path.join(base_dir, pkg, "rw.count") + if os.path.exists(count): + os.remove(count) + count = os.path.join(base_dir, pkg, "rw.count.lock") + if os.path.exists(count): + os.remove(count) + locks = os.path.join(base_dir, pkg, "locks") + if os.path.exists(locks): + shutil.rmtree(locks) + except Exception as e: + raise ConanException("Something went wrong while removing %s locks\n" + "Error: %s\n" + "Please clean your local conan cache manually" + % (pkg, str(e))) + out.warn("Migration: Removing old lock files finished\n") + + def migrate_to_default_profile(conf_path, default_profile_path): tag = "[settings_defaults]" old_conf = load(conf_path) From c76ffda0a26b473ba6feaa5391f108bddd17ecd2 Mon Sep 17 00:00:00 2001 From: James Date: Wed, 22 Nov 2017 17:27:20 +0100 Subject: [PATCH 3/7] workaround MSYS2 issue with psutil (#2060) --- conans/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/requirements.txt b/conans/requirements.txt index fe4873a38e1..59affd17e43 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -10,4 +10,4 @@ distro>=1.0.2, <1.1.0 pylint>=1.6.5, <=1.8.0 future==0.16.0 pygments>=2.0, <3.0 -psutil>=5.4.1 +psutil>=5.2.2 From e36af239f70fa6777479aa4155ef2a862995f22a Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Wed, 22 Nov 2017 17:43:22 +0100 Subject: [PATCH 4/7] High sierra support (#2062) --- conans/client/tools/oss.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conans/client/tools/oss.py b/conans/client/tools/oss.py index 2d3998b89e7..3c0ea08c803 100644 --- a/conans/client/tools/oss.py +++ b/conans/client/tools/oss.py @@ -179,6 +179,8 @@ def get_win_version_name(version): def get_osx_version_name(version): if not version: return None + elif version.minor() == "10.13.Z": + return "High Sierra" elif version.minor() == "10.12.Z": return "Sierra" elif version.minor() == "10.11.Z": From fa66f8bcc43e2a319d37f81e34e0177e5d8e485f Mon Sep 17 00:00:00 2001 From: Luis Martinez de Bartolome Izquierdo Date: Wed, 22 Nov 2017 18:55:37 +0100 Subject: [PATCH 5/7] Encoding issues (#2061) * 0.30.0-dev * encoding issues * fixed py2 --- conans/__init__.py | 1 + conans/server/conf/__init__.py | 11 +++++++++-- conans/test/util/tools_test.py | 16 +++++++--------- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/conans/__init__.py b/conans/__init__.py index 926ba0a0b58..9419572121f 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -17,3 +17,4 @@ SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, ] __version__ = '0.29.1' + diff --git a/conans/server/conf/__init__.py b/conans/server/conf/__init__.py index 5d6fff511a6..03114d6d8cc 100644 --- a/conans/server/conf/__init__.py +++ b/conans/server/conf/__init__.py @@ -1,7 +1,9 @@ """ Server's configuration variables """ +import six +from conans import tools from conans.util.env_reader import get_env from datetime import timedelta import os @@ -58,7 +60,11 @@ def _get_file_conf(self, section, varname=None): if not self._loaded: self._loaded = True - self.read(self.config_filename) + # To avoid encoding problems we use our tools.load + if six.PY3: + self.read_string(tools.load(self.config_filename)) + else: + self.read(self.config_filename) if varname: section = dict(self.items(section)) @@ -150,7 +156,8 @@ def validate_pass_encoding(password): try: password.encode('ascii') except (UnicodeDecodeError, UnicodeEncodeError): - raise ConanException("Password contains invalid characters. Only ASCII encoding is supported") + raise ConanException("Password contains invalid characters. " + "Only ASCII encoding is supported") return password if self.env_config["users"]: diff --git a/conans/test/util/tools_test.py b/conans/test/util/tools_test.py index 32b1e86d4d9..d54d9c1c204 100644 --- a/conans/test/util/tools_test.py +++ b/conans/test/util/tools_test.py @@ -53,15 +53,13 @@ def test_replace_in_file(self): replace_in_file(self.win_file, "nis", "nus") replace_in_file(self.bytes_file, "nis", "nus") - with open(self.win_file, "rt") as handler: - content = handler.read() - self.assertNotIn("nis", content) - self.assertIn("nus", content) - - with open(self.bytes_file, "rt") as handler: - content = handler.read() - self.assertNotIn("nis", content) - self.assertIn("nus", content) + content = tools.load(self.win_file) + self.assertNotIn("nis", content) + self.assertIn("nus", content) + + content = tools.load(self.bytes_file) + self.assertNotIn("nis", content) + self.assertIn("nus", content) class ToolsTest(unittest.TestCase): From fc0841fe5d047a8b22b5aa40c410e9bbb7d87227 Mon Sep 17 00:00:00 2001 From: James Date: Thu, 23 Nov 2017 09:55:09 +0100 Subject: [PATCH 6/7] reverted locks (#2068) --- conans/requirements.txt | 1 - conans/util/locks.py | 47 +++++++++++------------------------------ 2 files changed, 12 insertions(+), 36 deletions(-) diff --git a/conans/requirements.txt b/conans/requirements.txt index 59affd17e43..a77e87e8c8d 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -10,4 +10,3 @@ distro>=1.0.2, <1.1.0 pylint>=1.6.5, <=1.8.0 future==0.16.0 pygments>=2.0, <3.0 -psutil>=5.2.2 diff --git a/conans/util/locks.py b/conans/util/locks.py index fcfc25e6696..59a465655a8 100644 --- a/conans/util/locks.py +++ b/conans/util/locks.py @@ -2,8 +2,6 @@ from conans.util.log import logger import time from conans.util.files import save, load -import psutil -import os class NoLock(object): @@ -48,50 +46,29 @@ def _info_locked(self): self._output.info("If not the case, quit, and do 'conan remove %s -f'" % str(self._locked_item)) - def _pids(self): + def _readers(self): try: - contents = load(self._count_file) + return int(load(self._count_file)) except IOError: - return [] - else: - if not contents: - return [] - pids = [int(i) for i in contents.split(',')] - valid_pids = [] - for pid in pids: - if not psutil.pid_exists(abs(pid)): - lock_type = "write" if pid < 0 else "read" - self._output.info("invalidate %s lock from PID %s" % (lock_type, abs(pid))) - else: - valid_pids.append(pid) - return valid_pids + return 0 class ReadLock(Lock): - def _save_pids(self, pids): - save(self._count_file, ','.join(str(pid) for pid in pids)) - def __enter__(self): while True: with fasteners.InterProcessLock(self._count_lock_file, logger=logger): - pids = self._pids() - if len(pids) >= 0: - pids.append(os.getpid()) - save(self._count_file, ','.join(str(pid) for pid in pids)) + readers = self._readers() + if readers >= 0: + save(self._count_file, str(readers + 1)) break self._info_locked() time.sleep(READ_BUSY_DELAY) def __exit__(self, exc_type, exc_val, exc_tb): # @UnusedVariable with fasteners.InterProcessLock(self._count_lock_file, logger=logger): - pids = self._pids() - try: - pids.remove(os.getpid()) - except ValueError: - # legal situation - conan may remove directory with lock file during write - pass - self._save_pids(pids) + readers = self._readers() + save(self._count_file, str(readers - 1)) class WriteLock(Lock): @@ -99,13 +76,13 @@ class WriteLock(Lock): def __enter__(self): while True: with fasteners.InterProcessLock(self._count_lock_file, logger=logger): - pids = self._pids() - if not pids: - save(self._count_file, "-%s" % os.getpid()) + readers = self._readers() + if readers == 0: + save(self._count_file, "-1") break self._info_locked() time.sleep(WRITE_BUSY_DELAY) def __exit__(self, exc_type, exc_val, exc_tb): # @UnusedVariable with fasteners.InterProcessLock(self._count_lock_file, logger=logger): - save(self._count_file, "") + save(self._count_file, "0") From a6127944a046027563a7f9875bc6a25f4fea29bb Mon Sep 17 00:00:00 2001 From: James Date: Thu, 23 Nov 2017 10:06:48 +0100 Subject: [PATCH 7/7] Feature/0 29 1 migration (#2069) * reverted locks * extending the locks migration to 0.29 --- conans/client/migrations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conans/client/migrations.py b/conans/client/migrations.py index d13dc6f2661..e23ee5ad718 100644 --- a/conans/client/migrations.py +++ b/conans/client/migrations.py @@ -41,7 +41,7 @@ def _make_migrations(self, old_version): # VERSION 0.1 if old_version is None: return - if old_version < Version("0.29"): + if old_version < Version("0.29.1"): _migrate_lock_files(self.client_cache, self.out) old_settings = """ os: