diff --git a/.github/workflows/macOS.yml b/.github/workflows/macOS.yml index 4f0194ef5..aea9e2bbf 100644 --- a/.github/workflows/macOS.yml +++ b/.github/workflows/macOS.yml @@ -22,14 +22,14 @@ jobs: - name: Install X10 run: | - curl -sL https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip -o tensorflow-darwin-x64.zip + curl -sL https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip -o tensorflow-darwin-x64.zip unzip tensorflow-darwin-x64.zip - mv tensorflow-darwin-x64/Library/tensorflow-2.3.0 ~/Library/ + mv tensorflow-darwin-x64/Library/tensorflow-2.4.0 ~/Library/ - name: Build run: | - TOOLCHAINS=org.swift.50202012141a swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.3.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.3.0/usr/lib + TOOLCHAINS=org.swift.50202012141a swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.4.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.4.0/usr/lib # - name: Run tests # run: | - # DYLD_LIBRARY_PATH=${HOME}/Library/tensorflow-2.3.0/usr/lib TOOLCHAINS=org.swift.50202012141a swift test -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.3.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.3.0/usr/lib + # DYLD_LIBRARY_PATH=${HOME}/Library/tensorflow-2.4.0/usr/lib TOOLCHAINS=org.swift.50202012141a swift test -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -I${HOME}/Library/tensorflow-2.4.0/usr/include -Xlinker -L${HOME}/Library/tensorflow-2.4.0/usr/lib diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 778768b6c..ea1d7607c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -32,16 +32,16 @@ jobs: - name: Install X10 run: | - Invoke-WebRequest -UseBasicParsing -Uri https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip -OutFile tensorflow-windows-x64.zip + Invoke-WebRequest -UseBasicParsing -Uri https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip -OutFile tensorflow-windows-x64.zip Expand-Archive -Force -Path tensorflow-windows-x64.zip -DestinationPath C:\Library\ - Move-Item C:\Library\tensorflow-windows-x64\Library\tensorflow-2.3.0 C:\Library - echo "C:\Library\tensorflow-2.3.0\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + Move-Item C:\Library\tensorflow-windows-x64\Library\tensorflow-2.4.0 C:\Library + echo "C:\Library\tensorflow-2.4.0\usr\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - name: Build run: | # We build twice as the first one currently fails for some unexplained # reason. This should only build once. - swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.3.0\usr\include -Xlinker -LC:\Library\tensorflow-2.3.0\usr\lib - swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.3.0\usr\include -Xlinker -LC:\Library\tensorflow-2.3.0\usr\lib + swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.4.0\usr\include -Xlinker -LC:\Library\tensorflow-2.4.0\usr\lib + swift build -v -Xswiftc -DTENSORFLOW_USE_STANDARD_TOOLCHAIN -Xcc -IC:\Library\tensorflow-2.4.0\usr\include -Xlinker -LC:\Library\tensorflow-2.4.0\usr\lib # - name: Run tests # run: swift test -v diff --git a/CMakeLists.txt b/CMakeLists.txt index 7060d7424..5f13d5a5f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -167,7 +167,7 @@ if(NOT X10_FOUND AND NOT USE_BUNDLED_X10) GIT_REPOSITORY git://github.com/tensorflow/tensorflow GIT_TAG - r2.3 + r2.4 UPDATE_DISCONNECTED TRUE CONFIGURE_COMMAND @@ -207,6 +207,7 @@ if(NOT X10_FOUND AND NOT USE_BUNDLED_X10) set(X10_LIBRARY ${SOURCE_DIR}/bazel-bin/tensorflow/compiler/tf2xla/xla_tensor/${CMAKE_SHARED_LIBRARY_PREFIX}x10${CMAKE_SHARED_LIBRARY_SUFFIX}) set(X10_INCLUDE_DIRS ${SOURCE_DIR} + ${SOURCE_DIR}/tensorflow ${SOURCE_DIR}/bazel-bin ${SOURCE_DIR}/bazel-libtensorflow/external/com_google_absl ${SOURCE_DIR}/bazel-libtensorflow/external/com_google_protobuf/src diff --git a/Documentation/Development.md b/Documentation/Development.md index 125396f31..c3a23758a 100644 --- a/Documentation/Development.md +++ b/Documentation/Development.md @@ -409,8 +409,8 @@ $ swift test -Xcc -I/Library/tensorflow-2.4.0/usr/include -Xlinker -L/Library/te [swift]: https://swift.org/download/#snapshots [cmake]: https://www.cmake.org/download -[windows10]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip -[macOS]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80Mzc2OC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip +[windows10]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy13aW5kb3dzLXg2NA2/content?format=zip +[macOS]: https://artprodeus21.artifacts.visualstudio.com/A8fd008a0-56bc-482c-ba46-67f9425510be/3133d6ab-80a8-4996-ac4f-03df25cd3224/_apis/artifact/cGlwZWxpbmVhcnRpZmFjdDovL2NvbXBuZXJkL3Byb2plY3RJZC8zMTMzZDZhYi04MGE4LTQ5OTYtYWM0Zi0wM2RmMjVjZDMyMjQvYnVpbGRJZC80NTU3NC9hcnRpZmFjdE5hbWUvdGVuc29yZmxvdy1kYXJ3aW4teDY00/content?format=zip [bazel]: https://docs.bazel.build/versions/master/install.html [bazelisk]: https://github.com/bazelbuild/bazelisk [configure.py]: https://github.com/tensorflow/tensorflow/blob/master/configure.py diff --git a/Sources/CTensorFlow/CMakeLists.txt b/Sources/CTensorFlow/CMakeLists.txt index d597f86cc..9388d7cdf 100644 --- a/Sources/CTensorFlow/CMakeLists.txt +++ b/Sources/CTensorFlow/CMakeLists.txt @@ -12,13 +12,21 @@ list(GET X10_INCLUDE_DIRS 0 X10_SOURCE_DIR) install(FILES include/module.modulemap include/CTensorFlow.h DESTINATION lib/swift/tensorflow) -install(DIRECTORY ${X10_SOURCE_DIR}/tensorflow/c - DESTINATION lib/swift/tensorflow - FILES_MATCHING - PATTERN c_api.h - PATTERN c_api_experimental.h - PATTERN tf_attrtype.h - PATTERN tf_datatype.h - PATTERN tf_status.h - PATTERN tf_tensor.h - PATTERN eager/c_api.h) +install(FILES + ${X10_SOURCE_DIR}/tensorflow/c/c_api.h + ${X10_SOURCE_DIR}/tensorflow/c/c_api_experimental.h + ${X10_SOURCE_DIR}/tensorflow/c/c_api_macros.h + ${X10_SOURCE_DIR}/tensorflow/c/tf_attrtype.h + ${X10_SOURCE_DIR}/tensorflow/c/tf_datatype.h + ${X10_SOURCE_DIR}/tensorflow/c/tf_file_statistics.h + ${X10_SOURCE_DIR}/tensorflow/c/tf_status.h + ${X10_SOURCE_DIR}/tensorflow/c/tf_tensor.h + ${X10_SOURCE_DIR}/tensorflow/c/tf_tstring.h + DESTINATION lib/swift/tensorflow/c) +install(FILES + ${X10_SOURCE_DIR}/tensorflow/c/eager/c_api.h + DESTINATION lib/swift/tensorflow/c/eager) +install(FILES + ${X10_SOURCE_DIR}/tensorflow/core/platform/ctstring.h + ${X10_SOURCE_DIR}/tensorflow/core/platform/ctstring_internal.h + DESTINATION lib/swift/tensorflow/core/platform) diff --git a/Sources/TensorFlow/Core/StringTensor.swift b/Sources/TensorFlow/Core/StringTensor.swift index 6867fd1ee..b87a494fe 100644 --- a/Sources/TensorFlow/Core/StringTensor.swift +++ b/Sources/TensorFlow/Core/StringTensor.swift @@ -47,72 +47,23 @@ extension StringTensor { // utf8CString is null-terminated. TF APIs want the strings without null-terminators. let cStrings = scalars.map { $0.utf8CString.dropLast() } - // Note: `TENSORFLOW_MASTER` changes below are necessary for the new TensorFlow ABI-stable - // unified string tensor design. - #if TENSORFLOW_MASTER - let byteCount = scalars.count * MemoryLayout.stride - - let handle = TensorHandle( - shape: shape.dimensions, - byteCount: byteCount, - bufferInitializer: { tensorBuffer in - var dataAddr = - tensorBuffer.bindMemory(to: TF_TString.self, capacity: scalars.count) - for cString in cStrings { - TF_TString_Init(dataAddr) - cString.withUnsafeBufferPointer { buffer in - TF_TString_Copy(dataAddr, buffer.baseAddress, buffer.count) - } - dataAddr = dataAddr.advanced(by: 1) - } - }) - self.init(handle: handle) - #else - let tfEncodedSizes = cStrings.map { TF_StringEncodedSize($0.count) } - - // Format information copied from tensorflow/c/c_api.h: - // The format for TF_STRING tensors is: - // start_offset: array[uint64] - // data: byte[...] - // - // The string length (as a varint), followed by the contents of the string is encoded at - // data[start_offset[i]]]. - // The size of the "start_offset" region. - let startOffsetsByteCount = scalars.count * MemoryLayout.stride - - // The size of the "data" region. - let dataByteCount = tfEncodedSizes.reduce(0, +) * MemoryLayout.stride - - let handle = TensorHandle( - shape: shape.dimensions, - byteCount: startOffsetsByteCount + dataByteCount, - bufferInitializer: { tensorBuffer in - // Initialize the "start_offset" region. - var startOffset: UInt64 = 0 - var startOffsetAddr = - tensorBuffer.bindMemory(to: UInt64.self, capacity: scalars.count) - for tfEncodedSize in tfEncodedSizes { - startOffsetAddr.initialize(to: startOffset) - startOffsetAddr = startOffsetAddr.advanced(by: 1) - startOffset = startOffset + UInt64(tfEncodedSize) - } - - // Initialize the "data" region. - var dataAddr = tensorBuffer.advanced(by: startOffsetsByteCount) - .bindMemory(to: Int8.self, capacity: dataByteCount) - let status = TF_NewStatus() - for (cString, tfEncodedSize) in zip(cStrings, tfEncodedSizes) { - _ = cString.withUnsafeBufferPointer { buffer in - TF_StringEncode( - buffer.baseAddress, buffer.count, dataAddr, tfEncodedSize, status) - } - checkOk(status) - dataAddr = dataAddr.advanced(by: tfEncodedSize) + let byteCount = scalars.count * MemoryLayout.stride + + let handle = TensorHandle( + shape: shape.dimensions, + byteCount: byteCount, + bufferInitializer: { tensorBuffer in + var dataAddr = + tensorBuffer.bindMemory(to: TF_TString.self, capacity: scalars.count) + for cString in cStrings { + TF_TString_Init(dataAddr) + cString.withUnsafeBufferPointer { buffer in + TF_TString_Copy(dataAddr, buffer.baseAddress, buffer.count) } - TF_DeleteStatus(status) - }) - self.init(handle: handle) - #endif + dataAddr = dataAddr.advanced(by: 1) + } + }) + self.init(handle: handle) } /// Creates a 0-D `StringTensor` from a scalar value.