Skip to content

Commit

Permalink
(#33) Add DotNetZip to use for package extraction
Browse files Browse the repository at this point in the history 
This commit add DotNetZip to use for extracting nupkgs.
The System.IO.Packaging code is not able to extract files within
nupkgs that contain special characters like spaces or at symbols.
This causes those files to not be extracted from nupkgs when this
library is used by consumers like Chocolatey CLI. This will cause
prolems when packages packed with future versions of Chocolatey CLI
that use NuGet SDK assemblies as files with special characters are not
encoded due to changes in how NuGet operates, and therefore those
packages would be incompatible with older versions of Chocolatey CLI.
It also would effect the Community Repository services in a similar
way, as they also use this library.

This commit then switches the ZipPackage and OptimizedZipPackage
classes to use DotNetZip as the primary way to extract nupkgs. This
allows for the extraction of files with filenames that have special
characters. If DotNetZip is unable to read a passed in package, it
will fall back to the system Packaging code.

The build.cmd is updated to restore the DotNetZip package because
msbuild was not happy about restoreing it. The strongname script was
also updated to include the DotNetZip assembly in the ilmerged final
assembly so a specific reference to DotNetZip is not required in
projects that consume this library.
  • Loading branch information
@TheCakeIsNaOH
TheCakeIsNaOH committed Aug 30, 2022
1 parent 94fce3c commit ef9e97d
Show file tree
Hide file tree
Showing 7 changed files with 226 additions and 30 deletions.
1 change: 1 addition & 0 deletions build.cmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ if exist %nugetmsbuildpathtmp% set nugetmsbuildpath=%nugetmsbuildpathtmp%
set nugetmsbuildpathtmp="%ProgramFiles(x86)%\MSBuild\14.0\bin\msbuild"
if exist %nugetmsbuildpathtmp% set nugetmsbuildpath=%nugetmsbuildpathtmp%
set EnableNuGetPackageRestore=true
.\lib\NuGet.exe restore .\src\Core\packages.config -PackagesDirectory .\packages
%nugetmsbuildpath% Build\Build.CommandLine.proj /p:Configuration="%config%" /p:Platform="Any CPU" /p:TargetFrameworkVersion="v4.0" /m /v:M /fl /flp:LogFile=msbuild.log;Verbosity=Detailed /nr:false /target:GoMono

ENDLOCAL
4 changes: 2 additions & 2 deletions nuget/Chocolatey-NuGet.Core/strongname.cmd
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@ pause

mkdir %DIR%\output

%DIR%..\ILMerge\ILMerge.exe lib\net4\NuGet.Core.dll /keyfile:%DIR%..\..\chocolatey.snk /out:%DIR%\output\NuGet.Core.dll /targetplatform:v4,"%ProgramFiles(x86)%\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.0" /log:%DIR%ILMerge.DELETE.log /allowDup
REM %DIR%..\ILMerge\ILMerge.exe NuGet.Core.dll /keyfile:%DIR%..\..\chocolatey.snk /out:%DIR%NuGet.Core.dll /targetplatform:v4,"%ProgramFiles(x86)%\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.0" /log:%DIR%ILMerge.DELETE.log /ndebug /allowDup
%DIR%..\ILMerge\ILMerge.exe lib\net4\NuGet.Core.dll lib\net4\DotNetZip.dll /keyfile:%DIR%..\..\chocolatey.snk /out:%DIR%\output\NuGet.Core.dll /targetplatform:v4,"%ProgramFiles(x86)%\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.0" /log:%DIR%ILMerge.DELETE.log /allowDup
REM %DIR%..\ILMerge\ILMerge.exe NuGet.Core.dll DotNetZip.dll /keyfile:%DIR%..\..\chocolatey.snk /out:%DIR%NuGet.Core.dll /targetplatform:v4,"%ProgramFiles(x86)%\Reference Assemblies\Microsoft\Framework\.NETFramework\v4.0" /log:%DIR%ILMerge.DELETE.log /ndebug /allowDup

echo Copy the files from the output directory up to this directory and delete everything but this file, NuGet.Core.dll and NuGet.Core.pdb.
pause
3 changes: 3 additions & 0 deletions src/Core/Core.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,9 @@
<Compile Include="Utility\VersionSpec.cs" />
<Compile Include="Utility\WebRequestEventArgs.cs" />
<Compile Include="FileModifiers\XdtTransformer.cs" />
<Reference Include="DotNetZip, Version=1.16.0.0">
<HintPath>..\..\packages\DotNetZip.1.16.0\lib\net40\DotNetZip.dll</HintPath>
</Reference>
<Reference Include="Microsoft.Web.XmlTransform">
<HintPath>..\..\lib\Microsoft.Web.XmlTransform.dll</HintPath>
</Reference>
Expand Down
130 changes: 102 additions & 28 deletions src/Core/Packages/OptimizedZipPackage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
using System.Linq;
using System.Runtime.Versioning;
using NuGet.Resources;
using Ionic.Zip;

namespace NuGet
{
Expand Down Expand Up @@ -249,57 +250,130 @@ private void EnsurePackageFiles()
_expandedFolderPath = GetExpandedFolderPath();
}

using (Stream stream = GetStream())
string packageFullPath = _fileSystem.GetFullPath(_packagePath);
// This checks if the package is a zip package that DotNetZip is capable of reading.
// This normally should be the case in operation within Chocolatey,
// but when running some of the tests in this solution, a mock of a zip file is used, which DotNetZip is unable to handle.
// The old behavior is kept as a fallback if the package can't be read
// Use the package full path because DotNetZip is not happy about reading Nupkgs from a pre-existing stream for some reason.
if (ZipFile.IsZipFile(packageFullPath, false))
{
Package package = Package.Open(stream);
// unzip files inside package
var files = from part in package.GetParts()
where ZipPackage.IsPackageFile(part, package.PackageProperties.Identifier)
select part;
string packageId;

// now copy all package's files to disk
foreach (PackagePart file in files)
// Even if we are using DotNetZip, the package name is still needed, and the easiest way to get that is via the system Packaging code.
using (Stream stream = GetStream())
{
Package package = Package.Open(stream);
packageId = package.PackageProperties.Identifier;
package.Close();
}

// unzip files inside package
using (var zip = ZipFile.Read(packageFullPath))
{
string path = UriUtility.GetPath(file.Uri);
string filePath = Path.Combine(_expandedFolderPath, path);
zip.ExtractExistingFile = ExtractExistingFileAction.OverwriteSilently;

bool copyFile = true;
if (_expandedFileSystem.FileExists(filePath))
// now copy all package's files to disk
foreach (var file in zip)
{
using (Stream partStream = file.GetStream(),
targetStream = _expandedFileSystem.OpenFile(filePath))
if (file.IsDirectory) continue;
if (!ZipPackage.IsPackageFile(file.FileName, packageId)) continue;

//Normalizes path separator for each platform.
char separator = Path.DirectorySeparatorChar;
string path = Uri.UnescapeDataString(file.FileName).Replace('/', separator).Replace('\u005c', separator);
string filePath = Path.Combine(_expandedFolderPath, path);

bool copyFile = true;
if (_expandedFileSystem.FileExists(filePath))
{
// if the target file already exists,
// don't copy file if the lengths are equal.
copyFile = partStream.Length != targetStream.Length;
using (Stream targetStream = _expandedFileSystem.OpenFile(filePath))
{
// if the target file already exists,
// don't copy file if the lengths are equal.
copyFile = file.UncompressedSize != targetStream.Length;
}
}
}

if (copyFile)
{
using (Stream partStream = file.GetStream())
if (copyFile)
{
try
{
using (Stream targetStream = _expandedFileSystem.CreateFile(filePath))
{
partStream.CopyTo(targetStream);
file.Extract(targetStream);
}
}
catch (Exception)
{
// if the file is read-only or has an access denied issue, we just ignore it
}
}
}

var packageFile = new PhysicalPackageFile
var packageFile = new PhysicalPackageFile
{
SourcePath = _expandedFileSystem.GetFullPath(filePath),
TargetPath = path
};

_files[path] = packageFile;
}
}
}
else
{
using (Stream stream = GetStream())
{
Package package = Package.Open(stream);
// unzip files inside package
var files = from part in package.GetParts()
where ZipPackage.IsPackageFile(part, package.PackageProperties.Identifier)
select part;

// now copy all package's files to disk
foreach (PackagePart file in files)
{
SourcePath = _expandedFileSystem.GetFullPath(filePath),
TargetPath = path
};
string path = UriUtility.GetPath(file.Uri);
string filePath = Path.Combine(_expandedFolderPath, path);

bool copyFile = true;
if (_expandedFileSystem.FileExists(filePath))
{
using (Stream partStream = file.GetStream(),
targetStream = _expandedFileSystem.OpenFile(filePath))
{
// if the target file already exists,
// don't copy file if the lengths are equal.
copyFile = partStream.Length != targetStream.Length;
}
}

if (copyFile)
{
using (Stream partStream = file.GetStream())
{
try
{
using (Stream targetStream = _expandedFileSystem.CreateFile(filePath))
{
partStream.CopyTo(targetStream);
}
}
catch (Exception)
{
// if the file is read-only or has an access denied issue, we just ignore it
}
}
}

var packageFile = new PhysicalPackageFile
{
SourcePath = _expandedFileSystem.GetFullPath(filePath),
TargetPath = path
};

_files[path] = packageFile;
_files[path] = packageFile;
}
}
}
}
Expand Down
109 changes: 109 additions & 0 deletions src/Core/Packages/ZipPackage.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.IO.Packaging;
using System.Linq;
using System.Runtime.Versioning;
using Ionic.Zip;
using NuGet.Resources;

namespace NuGet
Expand Down Expand Up @@ -97,6 +98,56 @@ public override Stream GetStream()

public override void ExtractContents(IFileSystem fileSystem, string extractPath)
{
string packageId = "";
string packagePath = "";
bool hasFileStream = false;

// This checks if the filestream that the instance of this class has is a filestream.
// If it is not a filestream, then fall back to the system Packaging class to extract the nupkg
// This is because DotNetZip is not happy about reading Nupkgs from a pre-existing stream for some reason.
using (Stream stream = _streamFactory())
{
var filestream = stream as FileStream;
if (filestream != null)
{
hasFileStream = true;
packagePath = filestream.Name;

Package package = Package.Open(stream);
packageId = package.PackageProperties.Identifier;
package.Close();
}
}

// This checks if the package is a zip package that DotNetZip is capable of reading.
// This normally should be the case in operation within Chocolatey,
// but when running some of the tests in this solution, a mock of a zip file is used, which DotNetZip is unable to handle.
// The old behavior is kept as a fallback if the package can't be read.
// Use the package full path because DotNetZip is not happy about reading Nupkgs from a pre-existing stream for some reason.
if (hasFileStream && ZipFile.IsZipFile(packagePath, false))
{
using (var zip = ZipFile.Read(packagePath))
{
zip.ExtractExistingFile = ExtractExistingFileAction.OverwriteSilently;
foreach (var file in zip)
{
if (file.IsDirectory) continue;
if (!ZipPackage.IsPackageFile(file.FileName, packageId)) continue;

//Normalizes path separator for each platform.
char separator = Path.DirectorySeparatorChar;
string path = Uri.UnescapeDataString(file.FileName).Replace('/', separator).Replace('\u005c', separator);
string targetPath = Path.Combine(extractPath, path);

using (Stream targetStream = fileSystem.CreateFile(targetPath))
{
file.Extract(targetStream);
}
}
}
return;
}

using (Stream stream = _streamFactory())
{
var package = Package.Open(stream);
Expand Down Expand Up @@ -171,6 +222,54 @@ where IsAssemblyReference(file.Path)

private List<IPackageFile> GetFilesNoCache()
{
string packageId = "";
string packagePath = "";
bool hasFileStream = false;


// This checks if the filestream that the instance of this class has is a filestream.
// If it is not a filestream, then fall back to the system Packaging class to extract the nupkg
// This is because DotNetZip is not happy about reading Nupkgs from a pre-existing stream for some reason.
using (Stream stream = _streamFactory())
{
var filestream = stream as FileStream;
if (filestream != null)
{
hasFileStream = true;
packagePath = filestream.Name;

Package package = Package.Open(stream);
packageId = package.PackageProperties.Identifier;
package.Close();
}
}

// This checks if the package is a zip package that DotNetZip is capable of reading.
// This normally should be the case in operation within Chocolatey,
// but when running some of the tests in this solution, a mock of a zip file is used, which DotNetZip is unable to handle.
// The old behavior is kept as a fallback if the package can't be read.
// Use the package full path because DotNetZip is not happy about reading Nupkgs from a pre-existing stream for some reason.
if (hasFileStream && ZipFile.IsZipFile(packagePath, false))
{
var fileList = new List<IPackageFile>();

using (var zip = ZipFile.Read(packagePath))
{
foreach (var file in zip)
{
if (file.IsDirectory) continue;
if (!IsPackageFile(file.FileName, packageId)) continue;

char separator = Path.DirectorySeparatorChar;
var filename = Uri.UnescapeDataString(file.FileName).Replace('/', separator).Replace('\u005c', separator);

fileList.Add(new ZipPackageFile(filename, file.OpenReader()));
}
}

return fileList;
}

using (Stream stream = _streamFactory())
{
Package package = Package.Open(stream);
Expand Down Expand Up @@ -230,6 +329,16 @@ internal static bool IsPackageFile(PackagePart part, string packageId)
!PackageHelper.IsPackageManifest(path, packageId);
}

internal static bool IsPackageFile(string partPath, string packageId)
{
string directory = Path.GetDirectoryName(partPath);

// We exclude any opc files and the auto-generated package manifest file ({packageId}.nuspec)
return !ExcludePaths.Any(p => directory.StartsWith(p, StringComparison.OrdinalIgnoreCase)) &&
!PackageHelper.IsPackageManifest(partPath, packageId) &&
!string.Equals(partPath, "[Content_Types].xml", StringComparison.OrdinalIgnoreCase);
}

internal static void ClearCache(IPackage package)
{
var zipPackage = package as ZipPackage;
Expand Down
5 changes: 5 additions & 0 deletions src/Core/Packages/ZipPackageFile.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ internal class ZipPackageFile : IPackageFile
private readonly Func<Stream> _streamFactory;
private readonly FrameworkName _targetFramework;

public ZipPackageFile(string path, Stream stream)
: this(path, stream.ToStreamFactory())
{
}

public ZipPackageFile(PackagePart part)
: this(UriUtility.GetPath(part.Uri), part.GetStream().ToStreamFactory())
{
Expand Down
4 changes: 4 additions & 0 deletions src/Core/packages.config
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="dotnetzip" version="1.16.0" />
</packages>

0 comments on commit ef9e97d

Please sign in to comment.