Skip to content

Commit

Permalink
RUMM-1744 Add DirectoryMatcher type to distribution tool
Browse files Browse the repository at this point in the history
to simplify assertion of GH asset content
  • Loading branch information
ncreated committed Jan 3, 2022
1 parent 5fd9d56 commit c6a9c08
Show file tree
Hide file tree
Showing 3 changed files with 213 additions and 40 deletions.
117 changes: 77 additions & 40 deletions tools/distribution/src/assets/gh_asset.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,15 +11,8 @@
import glob
from tempfile import TemporaryDirectory, NamedTemporaryFile
from src.utils import remember_cwd, shell, read_sdk_version
from src.directory_matcher import DirectoryMatcher

SWIFT_CONTENT = [
'Datadog.xcframework',
'DatadogObjc.xcframework',
'DatadogCrashReporting.xcframework',
'Kronos.xcframework',
]
OBJC_CONTENT = ['CrashReporter.xcframework']
EXPECTED_ZIP_CONTENT = SWIFT_CONTENT + OBJC_CONTENT

class GHAsset:
"""
Expand All @@ -34,36 +27,27 @@ def __init__(self):

with NamedTemporaryFile(mode='w+', prefix='dd-gh-distro-', suffix='.xcconfig') as xcconfig:
xcconfig.write('BUILD_LIBRARY_FOR_DISTRIBUTION = YES\n')
xcconfig.seek(0) # without this line, content isn't actually written
xcconfig.seek(0) # without this line, content isn't actually written
os.environ['XCODE_XCCONFIG_FILE'] = xcconfig.name

# Produce XCFrameworks with carthage:
# - only checkout and `--no-build` as it will build in the next command:
shell('carthage bootstrap --platform iOS --no-build')
shell('carthage bootstrap --platform iOS --no-build', skip=True)
# - `--no-build` as it will build in the next command:
shell('carthage build --platform iOS --use-xcframeworks --no-use-binaries --no-skip-current')
shell('carthage build --platform iOS --use-xcframeworks --no-use-binaries --no-skip-current', skip=True)

# Create `.zip` archive:
zip_archive_name = f'Datadog-{read_sdk_version()}.zip'
with remember_cwd():
os.chdir('Carthage/Build')
shell(f'zip -q --symlinks -r {zip_archive_name} *.xcframework')
shell(f'zip -q --symlinks -r {zip_archive_name} *.xcframework', skip=True)

self.__path = f'{os.getcwd()}/Carthage/Build/{zip_archive_name}'
print(' → GH asset created')

def __repr__(self):
return f'[GHAsset: path = {self.__path}]'

def __content_with_swiftinterface(self, dir: str) -> set:
# e.g: /TMP_DIR/X.xcframework/ios-arm64/X.framework/Modules/X.swiftmodule/arm64.swiftinterface
swiftinterfaces = glob.iglob(f'{dir}/*.xcframework/**/*.framework/Modules/*.swiftmodule/*.swiftinterface', recursive=True)
# e.g: X.xcframework/ios-arm64/X.framework/Modules/X.swiftmodule/arm64.swiftinterface
relative_paths = [abs_path.removeprefix(dir + '/') for abs_path in swiftinterfaces]
# e.g: X.xcframework
product_names = [rel_path[0:rel_path.find('/')] for rel_path in relative_paths]
return set(product_names)

def validate(self, git_tag: str):
"""
Checks the `.zip` archive integrity with given `git_tag`.
Expand All @@ -79,27 +63,80 @@ def validate(self, git_tag: str):
# Inspect the content of zip archive:
with TemporaryDirectory() as unzip_dir:
shell(f'unzip -q {self.__path} -d {unzip_dir}')
actual_files = os.listdir(unzip_dir)
expected_files = EXPECTED_ZIP_CONTENT
actual_files.sort(), expected_files.sort()

if set(actual_files) != set(expected_files):
raise Exception(f'The content of `.zip` archive is not correct: \n'
f' - actual {actual_files}\n'
f' - expected: {expected_files}')

missing_swiftinterface_content = set(SWIFT_CONTENT).difference(self.__content_with_swiftinterface(unzip_dir))
if missing_swiftinterface_content:
raise Exception(f'Frameworks missing .swiftinterface: \n {missing_swiftinterface_content} \n')

print(f' → the content of `.zip` archive is correct: \n'
f' - actual: {actual_files}\n'
f' - expected: {expected_files}')

print(f' → details on bundled `XCFrameworks`:')
for file_path in glob.iglob(f'{unzip_dir}/*.xcframework/*', recursive=True):

print(f' → GH asset (zip) content:')
for file_path in glob.iglob(f'{unzip_dir}/**', recursive=True):
print(f' - {file_path.removeprefix(unzip_dir)}')

dm = DirectoryMatcher(path=unzip_dir)
dm.assert_number_of_files(expected_count=5)

dm.get(file='Datadog.xcframework').assert_it_has_files([
'ios-arm64',
'ios-arm64/BCSymbolMaps/*.bcsymbolmap',
'ios-arm64/dSYMs/*.dSYM',
'ios-arm64/**/arm64.swiftinterface',
'ios-arm64/**/arm64-apple-ios.swiftinterface',

'ios-arm64_x86_64-simulator',
'ios-arm64_x86_64-simulator/dSYMs/*.dSYM',
'ios-arm64_x86_64-simulator/**/arm64.swiftinterface',
'ios-arm64_x86_64-simulator/**/arm64-apple-ios-simulator.swiftinterface',
'ios-arm64_x86_64-simulator/**/x86_64.swiftinterface',
'ios-arm64_x86_64-simulator/**/x86_64-apple-ios-simulator.swiftinterface',
])

dm.get('DatadogObjc.xcframework').assert_it_has_files([
'ios-arm64',
'ios-arm64/BCSymbolMaps/*.bcsymbolmap',
'ios-arm64/dSYMs/*.dSYM',
'ios-arm64/**/arm64.swiftinterface',
'ios-arm64/**/arm64-apple-ios.swiftinterface',

'ios-arm64_x86_64-simulator',
'ios-arm64_x86_64-simulator/**/arm64.swiftinterface',
'ios-arm64_x86_64-simulator/**/arm64-apple-ios-simulator.swiftinterface',
'ios-arm64_x86_64-simulator/**/x86_64.swiftinterface',
'ios-arm64_x86_64-simulator/**/x86_64-apple-ios-simulator.swiftinterface',
])

dm.get('DatadogCrashReporting.xcframework').assert_it_has_files([
'ios-arm64',
'ios-arm64/BCSymbolMaps/*.bcsymbolmap',
'ios-arm64/**/arm64.swiftinterface',
'ios-arm64/**/arm64-apple-ios.swiftinterface',

'ios-x86_64-simulator',
'ios-x86_64-simulator/dSYMs/*.dSYM',
'ios-x86_64-simulator/**/x86_64.swiftinterface',
'ios-x86_64-simulator/**/x86_64-apple-ios-simulator.swiftinterface',
])

dm.get('CrashReporter.xcframework').assert_it_has_files([
'ios-arm64_arm64e_armv7_armv7s',
'ios-arm64_i386_x86_64-simulator',
])

dm.get('Kronos.xcframework').assert_it_has_files([
'ios-arm64_armv7',
'ios-arm64_armv7/BCSymbolMaps/*.bcsymbolmap',
'ios-arm64_armv7/dSYMs/*.dSYM',
'ios-arm64_armv7/**/arm.swiftinterface',
'ios-arm64_armv7/**/arm64-apple-ios.swiftinterface',
'ios-arm64_armv7/**/arm64.swiftinterface',
'ios-arm64_armv7/**/armv7-apple-ios.swiftinterface',
'ios-arm64_armv7/**/armv7.swiftinterface',

'ios-arm64_i386_x86_64-simulator',
'ios-arm64_i386_x86_64-simulator/dSYMs/*.dSYM',
'ios-arm64_i386_x86_64-simulator/**/arm64-apple-ios-simulator.swiftinterface',
'ios-arm64_i386_x86_64-simulator/**/i386-apple-ios-simulator.swiftinterface',
'ios-arm64_i386_x86_64-simulator/**/x86_64-apple-ios-simulator.swiftinterface',
'ios-arm64_i386_x86_64-simulator/**/x86_64.swiftinterface',
])

print(f' → the content of `.zip` archive is correct')

def publish(self, git_tag: str, overwrite_existing: bool, dry_run: bool):
"""
Uploads the `.zip` archive to GH Release for given `git_tag`.
Expand Down
43 changes: 43 additions & 0 deletions tools/distribution/src/directory_matcher.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
# -----------------------------------------------------------
# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
# This product includes software developed at Datadog (https://www.datadoghq.com/).
# Copyright 2019-2020 Datadog, Inc.
# -----------------------------------------------------------

import os
import glob


class DirectoryMatcherException(Exception):
pass


class DirectoryMatcher:
path: str

def __init__(self, path: str):
if os.path.exists(path):
self.path = path
else:
raise DirectoryMatcherException(f'Directory does not exist: {path}')

def assert_number_of_files(self, expected_count: int):
actual_count = len(os.listdir(self.path))
if expected_count != actual_count:
raise DirectoryMatcherException(f'Expected {expected_count} files in "{self.path}", but '
f'found {actual_count} instead.')

def assert_it_has_file(self, file_path: str):
search_path = os.path.join(self.path, file_path)
result = list(glob.iglob(search_path, recursive=True))

# if not os.path.exists(os.path.join(self.path, file_path)):
if not result:
raise DirectoryMatcherException(f'Expected "{self.path}" to include {file_path}, but it is missing.')

def assert_it_has_files(self, file_paths: [str]):
for file_path in file_paths:
self.assert_it_has_file(file_path)

def get(self, file: str) -> 'DirectoryMatcher':
return DirectoryMatcher(path=os.path.join(self.path, file))
93 changes: 93 additions & 0 deletions tools/distribution/tests/test_directory_matcher.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# -----------------------------------------------------------
# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
# This product includes software developed at Datadog (https://www.datadoghq.com/).
# Copyright 2019-2020 Datadog, Inc.
# -----------------------------------------------------------


import unittest
import os
from tempfile import TemporaryDirectory
from src.directory_matcher import DirectoryMatcher, DirectoryMatcherException


class DirectoryMatcherTestCase(unittest.TestCase):
def test_initializing(self):
with TemporaryDirectory() as tmp_dir:
self.assertEqual(DirectoryMatcher(path=tmp_dir).path, tmp_dir)

with self.assertRaises(DirectoryMatcherException):
_ = DirectoryMatcher(path=f'{tmp_dir}/non-existing-path')

def test_number_of_files(self):
with TemporaryDirectory() as tmp_dir:
dm = DirectoryMatcher(path=tmp_dir)
dm.assert_number_of_files(expected_count=0)

os.mkdir(os.path.join(tmp_dir, '1'))
os.mkdir(os.path.join(tmp_dir, '2'))
dm.assert_number_of_files(expected_count=2)

with self.assertRaises(DirectoryMatcherException):
dm.assert_number_of_files(expected_count=4)

def test_has_file(self):
with TemporaryDirectory() as tmp_dir:
dm = DirectoryMatcher(path=tmp_dir)
os.makedirs(os.path.join(tmp_dir, '1/1A/1AA.xyz'))
os.makedirs(os.path.join(tmp_dir, '1/1A/1AB.xyz'))
os.makedirs(os.path.join(tmp_dir, '2/2A/2AA/foo.xyz'))
os.makedirs(os.path.join(tmp_dir, '2/2A/2AB/foo.xyz'))

dm.assert_it_has_file('1')
dm.assert_it_has_file('1/1A/1AA.xyz')
dm.assert_it_has_file('1/1A/1AB.xyz')
dm.assert_it_has_file('**/1AA.xyz')
dm.assert_it_has_file('**/1AB.xyz')
dm.assert_it_has_file('**/*.xyz')
dm.assert_it_has_file('**/2A/**/*.xyz')
dm.assert_it_has_file('2')

with self.assertRaises(DirectoryMatcherException):
dm.assert_it_has_file('foo')

with self.assertRaises(DirectoryMatcherException):
dm.assert_it_has_file('1A')

def test_has_files(self):
with TemporaryDirectory() as tmp_dir:
dm = DirectoryMatcher(path=tmp_dir)
os.makedirs(os.path.join(tmp_dir, '1/1A/1AA.xyz'))
os.makedirs(os.path.join(tmp_dir, '1/1A/1AB.xyz'))
os.makedirs(os.path.join(tmp_dir, '2/2A/2AA/foo.xyz'))
os.makedirs(os.path.join(tmp_dir, '2/2A/2AB/foo.xyz'))

dm.assert_it_has_files([
'1',
'1/1A/1AA.xyz',
'1/1A/1AB.xyz',
'**/1AA.xyz',
'**/1AB.xyz',
'**/*.xyz',
'**/2A/**/*.xyz',
'2',
])

with self.assertRaises(DirectoryMatcherException):
dm.assert_it_has_files(file_paths=['foo', 'bar'])

with self.assertRaises(DirectoryMatcherException):
dm.assert_it_has_files(file_paths=['2', '**/foo'])

def test_get_submatcher(self):
with TemporaryDirectory() as tmp_dir:
os.makedirs(os.path.join(tmp_dir, '1/1A'))

dm = DirectoryMatcher(path=tmp_dir)
dm.assert_it_has_file('1')

dm = dm.get('1')
dm.assert_it_has_file('1A')

with self.assertRaises(DirectoryMatcherException):
dm.assert_it_has_file('1')

0 comments on commit c6a9c08

Please sign in to comment.