From 4be8e24ada8c48dfbd82d1fd2265f21fd88a779e Mon Sep 17 00:00:00 2001 From: "Ronald A. Richardson" Date: Thu, 10 Oct 2024 18:44:29 +0800 Subject: [PATCH] added ability to handle tar/gz bundle uploads, added endpoint to upload bundles via auth token --- .../Internal/v1/RegistryController.php | 154 +++++++++++ server/src/Models/RegistryExtensionBundle.php | 259 ++++++++++++++---- server/src/Models/RegistryUser.php | 5 + server/src/routes.php | 1 + 4 files changed, 369 insertions(+), 50 deletions(-) diff --git a/server/src/Http/Controllers/Internal/v1/RegistryController.php b/server/src/Http/Controllers/Internal/v1/RegistryController.php index db59d61..ceb14ff 100644 --- a/server/src/Http/Controllers/Internal/v1/RegistryController.php +++ b/server/src/Http/Controllers/Internal/v1/RegistryController.php @@ -5,7 +5,11 @@ use Fleetbase\Http\Controllers\Controller; use Fleetbase\Http\Resources\Category as CategoryResource; use Fleetbase\Models\Category; +use Fleetbase\Models\File; use Fleetbase\RegistryBridge\Models\RegistryExtension; +use Fleetbase\RegistryBridge\Models\RegistryExtensionBundle; +use Fleetbase\RegistryBridge\Models\RegistryUser; +use Fleetbase\RegistryBridge\Support\Utils; use Illuminate\Http\Request; class RegistryController extends Controller @@ -137,4 +141,154 @@ public function lookupPackage(Request $request) 'composer' => $composerJsonName, ]); } + + /** + * Handles the upload of an extension bundle to the registry. + * + * This method performs the following operations: + * - Authenticates the user using a Bearer token from the Authorization header. + * - Validates the uploaded bundle file (ensuring it's a valid tar.gz file). + * - Extracts necessary files (`extension.json`, `package.json`, `composer.json`) from the bundle. + * - Associates the bundle with the correct extension based on package information. + * - Checks if the user is authorized to upload bundles for the extension. + * - Uploads the bundle to the storage system. + * - Creates a file record in the database. + * - Updates metadata and versioning information. + * - Creates a new extension bundle record. + * + * @param Request $request the HTTP request containing the bundle file and authentication token + * + * @return \Illuminate\Http\JsonResponse a JSON response indicating the success or failure of the upload process + */ + public function bundleUpload(Request $request) + { + // Check for Authorization header + $authHeader = $request->header('Authorization'); + if (!$authHeader) { + return response()->json(['error' => 'Unauthorized.'], 401); + } + + // Extract the token from the 'Bearer ' prefix + $token = null; + if (preg_match('/Bearer\s(\S+)/', $authHeader, $matches)) { + $token = $matches[1]; + } + + // Validate the token (implement your own token validation logic) + $registryUser = RegistryUser::findFromToken($token); + if (!$registryUser) { + return response()->json(['error' => 'Unauthorized.', 'token' => $token], 401); + } + + // Check if file was uploaded + if (!$request->hasFile('bundle')) { + return response()->json(['error' => 'No bundle uploaded.'], 400); + } + + $bundle = $request->file('bundle'); + + // Validate the file + if (!$bundle->isValid()) { + return response()->json(['error' => 'Invalid bundle file uploaded.'], 400); + } + + // Ensure the file is a tar.gz + $mimeType = $bundle->getMimeType(); + if ($mimeType !== 'application/gzip' && $mimeType !== 'application/x-gzip') { + return response()->json(['error' => 'Invalid bundle file type.'], 400); + } + + // Get the extension assosciated to bundle by extension name + try { + $bundleData = RegistryExtensionBundle::extractUploadedBundleFile($bundle, ['extension.json', 'package.json', 'composer.json']); + } catch (\Throwable $e) { + return response()->json(['error' => $e->getMessage()], 400); + } + + $bundlePackageData = Utils::getObjectKeyValue($bundleData, 'package.json') ?? Utils::getObjectKeyValue($bundleData, 'composer.json'); + if ($bundlePackageData && data_get($bundlePackageData, 'name')) { + $extension = RegistryExtension::findByPackageName(data_get($bundlePackageData, 'name')); + if (!$extension) { + return response()->json(['error' => 'Unable to find extension for the uploaded bundle.'], 400); + } + + if ($extension->company_uuid !== $registryUser->company_uuid) { + return response()->json(['error' => 'User is not authorized to upload bundles for this extension.'], 401); + } + } else { + return response()->json(['error' => 'Unable to parse uploaded bundle.'], 400); + } + + // Prepare to upload the bundle + $size = $bundle->getSize(); + $fileName = File::randomFileNameFromRequest($request, 'bundle'); + $disk = config('filesystems.default'); + $bucket = config('filesystems.disks.' . $disk . '.bucket', config('filesystems.disks.s3.bucket')); + $path = 'uploads/extensions/' . $extension->uuid . '/bundles'; + $type = 'extension_bundle'; + + // Upload the bundle + try { + $path = $bundle->storeAs($path, $fileName, ['disk' => $disk]); + } catch (\Throwable $e) { + return response()->error($e->getMessage()); + } + + // If file upload failed + if ($path === false) { + return response()->error('File upload failed.'); + } + + // Create a file record + try { + $file = File::createFromUpload($request->file('bundle'), $path, $type, $size, $disk, $bucket); + } catch (\Throwable $e) { + return response()->error($e->getMessage()); + } + + // Set company and uploader + $file->update([ + 'company_uuid' => $registryUser->company_uuid, + 'uploader_uuid' => $registryUser->user_uuid, + ]); + + // Set file subject to extension + $file = $file->setSubject($extension); + + // Get extension.json contents + $extensionJson = Utils::getObjectKeyValue($bundleData, 'extension.json'); + if (!$extensionJson) { + return response()->error('Unable to find `extension.json` file required in bundle.'); + } + + // Set version in file meta + $file->updateMeta('version', data_get($extensionJson, 'version')); + + // Check if version is set + if (!isset($extensionJson->version)) { + return response()->error('No `version` set in the `extension.json`'); + } + + // Check if either api or engine property is set + if (!isset($extensionJson->engine) && !isset($extensionJson->api)) { + return response()->error('No `api` or `engine` property set in the `extension.json`'); + } + + // Set bundle number to parsed JSON + $extensionJson->bundle_number = RegistryExtensionBundle::getNextBundleNumber($extension); + + // Create the bundle + $extensionBundle = RegistryExtensionBundle::create([ + 'company_uuid' => $registryUser->company_uuid, + 'created_by_uuid' => $registryUser->user_uuid, + 'extension_uuid' => $extension->uuid, + 'bundle_uuid' => $file->uuid, + 'status' => 'pending', + ]); + + $extensionBundle->update(['bundle_number' => $extensionJson->bundle_number, 'version' => $extensionJson->version]); + $extensionBundle->updateMetaProperties((array) $bundleData); + + return response()->json(['message' => 'Bundle uploaded successfully', 'filename' => $fileName, 'bundle' => $extensionBundle, 'extension' => $extension], 200); + } } diff --git a/server/src/Models/RegistryExtensionBundle.php b/server/src/Models/RegistryExtensionBundle.php index d76832c..5cdfb96 100644 --- a/server/src/Models/RegistryExtensionBundle.php +++ b/server/src/Models/RegistryExtensionBundle.php @@ -15,6 +15,8 @@ use Fleetbase\Traits\HasPublicId; use Fleetbase\Traits\HasUuid; use Fleetbase\Traits\Searchable; +use Illuminate\Database\Eloquent\Relations\BelongsTo; +use Illuminate\Http\UploadedFile; use Illuminate\Support\Facades\Storage; use Illuminate\Support\Str; use stdClass; @@ -111,44 +113,30 @@ protected static function boot() }); } - /** - * @return \Illuminate\Database\Eloquent\Relations\BelongsTo - */ - public function company() + public function company(): BelongsTo { return $this->belongsTo(Company::class, 'company_uuid', 'uuid'); } - /** - * @return \Illuminate\Database\Eloquent\Relations\BelongsTo - */ - public function createdBy() + public function createdBy(): BelongsTo { return $this->belongsTo(User::class, 'created_by_uuid', 'uuid'); } - /** - * @return \Illuminate\Database\Eloquent\Relations\BelongsTo - */ - public function extension() + public function extension(): BelongsTo { return $this->belongsTo(RegistryExtension::class, 'extension_uuid', 'uuid'); } - /** - * @return \Illuminate\Database\Eloquent\Relations\BelongsTo - */ - public function bundle() + public function bundle(): BelongsTo { return $this->belongsTo(File::class, 'bundle_uuid', 'uuid'); } /** * Get the bundle original filename. - * - * @return string */ - public function getBundleFilenameAttribute() + public function getBundleFilenameAttribute(): ?string { if ($this->bundle instanceof File) { return $this->bundle->original_filename; @@ -168,7 +156,7 @@ public function getBundleFilenameAttribute() * * @return string a unique, uppercase bundle ID */ - public static function generateUniqueBundleId() + public static function generateUniqueBundleId(): string { do { $prefix = 'EXTBNDL'; @@ -204,7 +192,7 @@ public static function generateUniqueBundleId() protected static function extractBundleFile(File $bundle, $filenames = 'extension.json', $options = []): ?\stdClass { $shouldParseJson = data_get($options, 'parse_json', true); - $tempDir = sys_get_temp_dir() . '/' . str_replace(['.', ','], '_', uniqid('fleetbase_zip_', true)); + $tempDir = sys_get_temp_dir() . '/' . str_replace(['.', ','], '_', uniqid('fleetbase_archive_', true)); mkdir($tempDir); // Download the file to a local temporary directory @@ -213,7 +201,7 @@ protected static function extractBundleFile(File $bundle, $filenames = 'extensio file_put_contents($tempFilePath, $contents); // Extract file paths - $extractedFilePaths = static::_extractAndFindFile($tempFilePath, $tempDir, $filenames); + $extractedFilePaths = static::_extractAndFindFile($tempFilePath, $tempDir, $filenames, $bundle->getExtension()); $result = new \stdClass(); foreach ($extractedFilePaths as $filename => $path) { if (file_exists($path)) { @@ -238,46 +226,177 @@ protected static function extractBundleFile(File $bundle, $filenames = 'extensio } /** - * Extracts and finds the path(s) of specified file(s) within a zipped archive. + * Extracts specified files from an uploaded bundle file and optionally parses their contents. * - * Opens the specified ZIP archive and extracts it to a temporary directory. It then - * searches for the given file(s) in both the root and the first valid subdirectory of - * the unzipped content. It returns an associative array of file paths, indexed by filenames. - * Invalid directories such as '__MACOSX', '.', '..', etc., are excluded from the search. + * This method handles the extraction of files from an uploaded archive (ZIP or TAR.GZ) and returns their contents. * - * @param string $zipFilePath path to the ZIP archive file - * @param string $tempDir temporary directory where the ZIP file is extracted - * @param string|array $targetFiles a single filename or an array of filenames to search for within the archive + * @param UploadedFile $bundle the uploaded bundle file to extract + * @param string|array $filenames The filename or array of filenames to extract from the bundle (default: 'extension.json'). + * @param array $options additional options: + * - 'parse_json' (bool): Whether to parse the file contents as JSON (default: true) * - * @return array associative array of paths for the requested files within the archive + * @return \stdClass|null an object containing the extracted file contents, or null if extraction fails + * + * @throws \Exception if extraction of the bundle fails */ - private static function _extractAndFindFile($zipFilePath, $tempDir, $targetFiles) + public static function extractUploadedBundleFile(UploadedFile $bundle, $filenames = 'extension.json', $options = []): ?\stdClass + { + $shouldParseJson = data_get($options, 'parse_json', true); + $tempDir = sys_get_temp_dir() . '/' . str_replace(['.', ','], '_', uniqid('fleetbase_archive_', true)); + mkdir($tempDir); + + // Download the file to a local temporary directory + $tempFilePath = $bundle->getPathname(); + + // Get the archive extension + $extension = $bundle->guessExtension(); + if ($extension === 'gz') { + $extension = 'tar.gz'; + } + + // Extract file paths + $extractedFilePaths = static::_extractAndFindFile($tempFilePath, $tempDir, $filenames, $extension); + $result = new \stdClass(); + foreach ($extractedFilePaths as $filename => $path) { + if (file_exists($path)) { + $fileContents = file_get_contents($path); + if ($shouldParseJson) { + $result->$filename = json_decode($fileContents); + } else { + $result->$filename = $fileContents; + } + } + } + + // Cleanup: Delete the temporary directory + try { + array_map('unlink', glob("$tempDir/*.*")); + } catch (\Throwable $e) { + // Probably a directory ... + } + Utils::deleteDirectory($tempDir); + + return $result; + } + + /** + * Extracts target files from an archive and finds their paths. + * + * This method handles both ZIP and TAR.GZ archives. It extracts the archive to a temporary directory + * and searches for the specified target files within the extracted contents. + * + * @param string $archiveFilePath the full path to the archive file to extract + * @param string $tempDir the temporary directory where the archive will be extracted + * @param string|array $targetFiles the filename or array of filenames to search for within the extracted contents + * @param string $extension the expected extension of the archive file (default: 'zip') + * + * @return array an associative array where keys are target filenames and values are their full paths within the extracted contents + * + * @throws \Exception if the archive format is unsupported or extraction fails + */ + private static function _extractAndFindFile($archiveFilePath, $tempDir, $targetFiles, $extension = 'zip'): array { $paths = []; - $zip = new \ZipArchive(); - if ($zip->open($zipFilePath) === true) { + // Ensure $tempDir exists + if (!is_dir($tempDir)) { + mkdir($tempDir, 0777, true); + } + + // Get the base name of the archive file with extension + $archiveFileName = basename($archiveFilePath); + + // Determine the extension of the archive file + $archiveExtension = pathinfo($archiveFileName, PATHINFO_EXTENSION); + if (empty($archiveExtension)) { + // Try to determine the extension from the MIME type + $finfo = new \finfo(FILEINFO_MIME_TYPE); + $mimeType = $finfo->file($archiveFilePath); + + if ($mimeType === 'application/zip') { + $archiveExtension = 'zip'; + } elseif ($mimeType === 'application/gzip' || $mimeType === 'application/x-gzip') { + $archiveExtension = 'gz'; + } else { + $archiveExtension = $extension; // Default to 'zip' if not detected + } + } + + // Create a temporary file in $tempDir with the correct extension + $tempArchivePath = $tempDir . '/' . uniqid('archive_', false) . '.' . $archiveExtension; + + // Copy the archive file to the temporary file + copy($archiveFilePath, $tempArchivePath); + + $zip = new \ZipArchive(); + + if ($zip->open($tempArchivePath) === true) { + // If it's a ZIP file, extract it $zip->extractTo($tempDir); $zip->close(); + // Remove the temp archive file + unlink($tempArchivePath); + } else { + // Attempt to handle as tar.gz or tar using system 'tar' command + try { + // Determine the appropriate flags for tar extraction + $flags = ''; + if (in_array($archiveExtension, ['gz', 'tgz', 'tar.gz'])) { + // For .tar.gz or .tgz files + $flags = '-xzf'; + } elseif ($archiveExtension === 'tar') { + // For .tar files + $flags = '-xf'; + } else { + // Unsupported archive format + unlink($tempArchivePath); + throw new \Exception('Unsupported archive format.'); + } - foreach ((array) $targetFiles as $targetFile) { - // Direct check in the tempDir - $directPath = $tempDir . '/' . $targetFile; - if (file_exists($directPath)) { - $paths[$targetFile] = $directPath; - continue; + // Build the command safely using escapeshellarg + $command = sprintf('tar %s %s -C %s', $flags, escapeshellarg($tempArchivePath), escapeshellarg($tempDir)); + + // Execute the command + $output = []; + $returnVar = 0; + exec($command, $output, $returnVar); + + // Check if the command was successful + if ($returnVar !== 0) { + throw new \Exception('Extraction failed with code ' . $returnVar . ': ' . implode("\n", $output)); } - // Check in the first subdirectory - $files = scandir($tempDir); - foreach ($files as $file) { - $invalidDirectories = ['__MACOSX', '.', '..', 'DS_Store', '.DS_Store', '.idea', '.vscode']; - if (!Str::startsWith($file, ['.', '_']) && !in_array($file, $invalidDirectories) && is_dir($tempDir . '/' . $file)) { - $nestedPath = $tempDir . '/' . $file . '/' . $targetFile; - if (file_exists($nestedPath)) { - $paths[$targetFile] = $nestedPath; - break; - } + // Clean up temporary archive file + unlink($tempArchivePath); + } catch (\Exception $e) { + // Handle extraction errors + // Clean up temporary files + if (file_exists($tempArchivePath)) { + unlink($tempArchivePath); + } + // Throw the exception + throw new \Exception('Failed to extract archive: ' . $e->getMessage()); + } + } + + // Now search for the target files in the extracted contents + foreach ((array) $targetFiles as $targetFile) { + // Direct check in the tempDir + $directPath = $tempDir . '/' . $targetFile; + if (file_exists($directPath)) { + $paths[$targetFile] = $directPath; + continue; + } + + // Check in the first subdirectory + $files = scandir($tempDir); + foreach ($files as $file) { + $invalidDirectories = ['__MACOSX', '.', '..', 'DS_Store', '.DS_Store', '.idea', '.vscode']; + if (!in_array($file, $invalidDirectories) && is_dir($tempDir . '/' . $file)) { + $nestedPath = $tempDir . '/' . $file . '/' . $targetFile; + if (file_exists($nestedPath)) { + $paths[$targetFile] = $nestedPath; + break; } } } @@ -673,6 +792,12 @@ public function buildConsole() } } + /** + * Simulates the installation progress of an extension by publishing progress updates. + * + * This method publishes progress updates to a SocketCluster channel for each step in the installation process. + * It can be used to provide real-time feedback to clients about the installation status. + */ public function runInstallerProgress(): void { $channel = implode('.', ['install', $this->company_uuid, $this->extension_uuid]); @@ -693,6 +818,12 @@ public function runInstallerProgress(): void } } + /** + * Simulates the uninstallation progress of an extension by publishing progress updates. + * + * This method publishes progress updates to a SocketCluster channel for each step in the uninstallation process. + * It can be used to provide real-time feedback to clients about the uninstallation status. + */ public function runUninstallerProgress(): void { $channel = implode('.', ['uninstall', $this->company_uuid, $this->extension_uuid]); @@ -1012,4 +1143,32 @@ public static function pnpmBuildOutputProgress($output): int // Default progress if no known phrases are matched return 0; } + + /** + * Calculates the next bundle number for a given extension. + * + * This method counts the existing bundles associated with the extension and returns the next sequential number. + * + * @param string|RegistryExtension $extension the UUID of the extension or a RegistryExtension instance + * + * @return int the next bundle number for the extension + */ + public static function getNextBundleNumber(string|RegistryExtension $extension): int + { + $numberOfBundles = RegistryExtensionBundle::whereHas('extension', function ($query) use ($extension) { + $query->where('uuid', Str::isUuid($extension) ? $extension : $extension->uuid); + })->count(); + + return ($numberOfBundles ?? 0) + 1; + } + + /** + * Retrieves the next bundle number for the current extension instance. + * + * @return int the next bundle number + */ + public function nextBundleNumber() + { + return static::getNextBundleNumber($this->extension_uuid); + } } diff --git a/server/src/Models/RegistryUser.php b/server/src/Models/RegistryUser.php index ba898c0..64dbc93 100644 --- a/server/src/Models/RegistryUser.php +++ b/server/src/Models/RegistryUser.php @@ -212,6 +212,11 @@ public static function findFromUsername(string $username): ?RegistryUser ->first(); } + public static function findFromToken(string $token): ?RegistryUser + { + return static::where('registry_token', $token)->orWhere('token', $token)->first(); + } + /** * Determine if the registry user can access a specific package. * diff --git a/server/src/routes.php b/server/src/routes.php index e4929ff..4c28b76 100644 --- a/server/src/routes.php +++ b/server/src/routes.php @@ -14,6 +14,7 @@ */ // Lookup package endpoint Route::get(config('internals.api.routing.prefix', '~registry') . '/v1/lookup', 'Fleetbase\RegistryBridge\Http\Controllers\Internal\v1\RegistryController@lookupPackage'); +Route::post(config('internals.api.routing.prefix', '~registry') . '/v1/bundle-upload', 'Fleetbase\RegistryBridge\Http\Controllers\Internal\v1\RegistryController@bundleUpload'); Route::prefix(config('internals.api.routing.prefix', '~registry'))->middleware(['fleetbase.registry'])->namespace('Fleetbase\RegistryBridge\Http\Controllers')->group( function ($router) { /*