Implement sync proxy metadata command

ansible/roles/cron/tasks/main.yml

@@ -5,6 +5,13 @@
     minute="*/5"
     job="{{ app_root }}/bin/console repman:proxy:sync-releases >>{{ app_root }}/var/log/cron.log 2>&1"
 
+- name: Add sync proxy metadata command to cron
+  cron:
+    name="Sync proxy metadata with origins"
+    user="{{ system_user }}"
+    minute="*/6"
+    job="{{ app_root }}/bin/console repman:proxy:sync-metadata >>{{ app_root }}/var/log/cron.log 2>&1"
+
 - name: Add security update-db command to cron
   cron:
     name="Update security advisories database"

composer.json

@@ -71,6 +71,7 @@
         "ekino/phpstan-banned-code": "^0.3.1",
         "friendsofphp/php-cs-fixer": "^2.16",
         "fzaninotto/faker": "^1.9",
+        "league/flysystem-memory": "^1.0",
         "phpstan/extension-installer": "^1.0",
         "phpstan/phpstan": "^0.12.18",
         "phpstan/phpstan-deprecation-rules": "^0.12.2",

docker/crontabs/root

@@ -1,2 +1,3 @@
 */5 * * * * /app/bin/console repman:proxy:sync-releases
+*/6 * * * * /app/bin/console repman:proxy:sync-metadata
 0 */2 * * * /app/bin/console repman:security:update-db

src/Command/ProxySyncMetadataCommand.php

@@ -0,0 +1,62 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Buddy\Repman\Command;
+
+use Buddy\Repman\Service\Downloader;
+use Buddy\Repman\Service\Proxy;
+use Buddy\Repman\Service\Proxy\ProxyRegister;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Lock\LockFactory;
+
+final class ProxySyncMetadataCommand extends Command
+{
+    public const LOCK_TTL = 60;
+    public const LOCK_NAME = 'proxy_metadata';
+
+    private ProxyRegister $register;
+    private Downloader $downloader;
+    private LockFactory $lockFactory;
+
+    public function __construct(ProxyRegister $register, Downloader $downloader, LockFactory $lockFactory)
+    {
+        $this->register = $register;
+        $this->downloader = $downloader;
+        $this->lockFactory = $lockFactory;
+
+        parent::__construct();
+    }
+
+    /**
+     * @return void
+     */
+    protected function configure()
+    {
+        $this
+            ->setName('repman:proxy:sync-metadata')
+            ->setDescription('Sync proxy metadata with origins')
+        ;
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $lock = $this->lockFactory->createLock(self::LOCK_NAME, self::LOCK_TTL);
+        if (!$lock->acquire()) {
+            return 0;
+        }
+
+        try {
+            $this->register->all()->forEach(function (Proxy $proxy) use ($lock): void {
+                $proxy->syncMetadata();
+                $lock->refresh();
+            });
+        } finally {
+            $lock->release();
+        }
+
+        return 0;
+    }
+}

src/Service/Downloader.php

@@ -14,4 +14,9 @@ interface Downloader
      * @return Option<string>
      */
     public function getContents(string $url, array $headers = [], callable $notFoundHandler = null): Option;
+
+    /**
+     * @return Option<int>
+     */
+    public function getLastModified(string $url): Option;
 }

src/Service/Downloader/NativeDownloader.php

@@ -33,6 +33,19 @@ public function getContents(string $url, array $headers = [], callable $notFound
         return Option::none();
     }
 
+    /**
+     * @return Option<int>
+     */
+    public function getLastModified(string $url): Option
+    {
+        $headers = @get_headers($url, 1, stream_context_create(['http' => ['method' => 'HEAD']]));
+        if (!is_array($headers) || !isset($headers['Last-Modified'])) {
+            return Option::none();
+        }
+
+        return Option::some((int) strtotime($headers['Last-Modified']));
+    }
+
     /**
      * @param string[] $headers
      *

src/Service/Proxy.php

@@ -92,8 +92,9 @@ public function download(string $package, string $version): void
 
         foreach ($this->decodeMetadata($package) as $packageData) {
             $lastDist = $packageData['dist'] ?? $lastDist;
-            if ($version === $packageData['version']) {
-                $this->filesystem->write($this->distPath($package, $lastDist['reference'], $lastDist['type']), $this->downloader->getContents($lastDist['url'])
+            $path = $this->distPath($package, $lastDist['reference'], $lastDist['type']);
+            if ($version === $packageData['version'] && !$this->filesystem->has($path)) {
+                $this->filesystem->write($path, $this->downloader->getContents($lastDist['url'])
                     ->getOrElseThrow(new \RuntimeException(sprintf('Failed to download file from %s', $lastDist['url'])))
                 );
                 break;
@@ -110,6 +111,39 @@ public function removeDist(string $package): void
         $this->filesystem->deleteDir(sprintf('%s/dist/%s', $this->name, $package));
     }
 
+    public function syncMetadata(): void
+    {
+        foreach ($this->filesystem->listContents($this->name) as $dir) {
+            if (!in_array($dir['basename'], ['p', 'p2'], true)) {
+                continue;
+            }
+
+            $this->syncPackagesMetadata(array_filter(
+                $this->filesystem->listContents($dir['path'], true),
+                fn (array $file) => $file['type'] === 'file' && $file['extension'] === 'json')
+            );
+        }
+    }
+
+    /**
+     * @param mixed[] $files
+     */
+    private function syncPackagesMetadata(array $files): void
+    {
+        foreach ($files as $file) {
+            $url = sprintf('%s://%s', parse_url($this->url, PHP_URL_SCHEME), $file['path']);
+            // todo: what if proxy do not return `Last-Modified` header?
+            if ($this->downloader->getLastModified($url)->getOrElse(0) > ($file['timestamp'] ?? time())) {
+                $metadata = $this->downloader->getContents($url)->getOrNull();
+                if ($metadata === null) {
+                    continue;
+                }
+
+                $this->filesystem->put($file['path'], $metadata);
+            }
+        }
+    }
+
     /**
      * @return mixed[]
      */

symfony.lock

@@ -229,6 +229,9 @@
             "var/storage/.gitignore"
         ]
     },
+    "league/flysystem-memory": {
+        "version": "1.0.2"
+    },
     "league/oauth2-client": {
         "version": "2.4.1"
     },

tests/Doubles/FakeDownloader.php

@@ -9,8 +9,14 @@
 
 final class FakeDownloader implements Downloader
 {
+    // todo: remove this and allow only for in-memory content to explicit control
     private string $basePath;
 
+    /**
+     * @var mixed[]
+     */
+    private array $content = [];
+
     public function __construct()
     {
         $this->basePath = __DIR__.'/../Resources';
@@ -23,6 +29,10 @@ public function __construct()
      */
     public function getContents(string $url, array $headers = [], callable $notFoundHandler = null): Option
     {
+        if (isset($this->content[$url])) {
+            return Option::some($this->content[$url]['content']);
+        }
+
         $path = $this->basePath.parse_url($url, PHP_URL_PATH);
 
         if (file_exists($path)) {
@@ -35,4 +45,29 @@ public function getContents(string $url, array $headers = [], callable $notFound
 
         return Option::none();
     }
+
+    /**
+     * @return Option<int>
+     */
+    public function getLastModified(string $url): Option
+    {
+        if (isset($this->content[$url])) {
+            return Option::some($this->content[$url]['timestamp']);
+        }
+
+        $path = $this->basePath.parse_url($url, PHP_URL_PATH);
+        if (file_exists($path)) {
+            return Option::some((int) filemtime($path));
+        }
+
+        return Option::none();
+    }
+
+    public function addContent(string $url, ?string $content, int $timestamp = null): void
+    {
+        $this->content[$url] = [
+            'timestamp' => $timestamp ?? time(),
+            'content' => $content,
+        ];
+    }
 }

tests/Functional/Command/ProxySyncMetadataCommandTest.php

@@ -0,0 +1,64 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Buddy\Repman\Tests\Functional\Command;
+
+use Buddy\Repman\Command\ProxySyncMetadataCommand;
+use Buddy\Repman\Service\Proxy\ProxyRegister;
+use Buddy\Repman\Tests\Doubles\FakeDownloader;
+use Buddy\Repman\Tests\Functional\FunctionalTestCase;
+use Symfony\Component\Console\Tester\CommandTester;
+use Symfony\Component\Lock\LockFactory;
+use Symfony\Component\Lock\Store\FlockStore;
+
+final class ProxySyncMetadataCommandTest extends FunctionalTestCase
+{
+    private FakeDownloader $downloader;
+    private LockFactory $lockFactory;
+
+    protected function setUp(): void
+    {
+        parent::setUp();
+        $this->downloader = new FakeDownloader();
+        $this->lockFactory = new LockFactory(new FlockStore());
+    }
+
+    public function testMetadataSynchronization(): void
+    {
+        $lock = $this->lockFactory->createLock(ProxySyncMetadataCommand::LOCK_NAME);
+        self::assertFalse($lock->isAcquired());
+
+        $commandTester = new CommandTester($this->prepareCommand());
+        $result = $commandTester->execute([]);
+
+        self::assertEquals($result, 0);
+
+        // test if lock was released
+        self::assertTrue($lock->acquire());
+        $lock->release();
+    }
+
+    public function testJobLocking(): void
+    {
+        $lock = $this->lockFactory->createLock(ProxySyncMetadataCommand::LOCK_NAME);
+        $lock->acquire();
+        self::assertTrue($lock->isAcquired());
+
+        $commandTester = new CommandTester($this->prepareCommand());
+        $result = $commandTester->execute([]);
+
+        self::assertEquals($result, 0);
+        self::assertTrue($lock->isAcquired());
+        $lock->release();
+    }
+
+    private function prepareCommand(): ProxySyncMetadataCommand
+    {
+        return new ProxySyncMetadataCommand(
+            $this->container()->get(ProxyRegister::class),
+            $this->downloader,
+            $this->lockFactory
+        );
+    }
+}

tests/Unit/DistTest.php

@@ -0,0 +1,18 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Buddy\Repman\Tests\Unit;
+
+use Buddy\Repman\Service\Dist;
+use PHPUnit\Framework\TestCase;
+
+final class DistTest extends TestCase
+{
+    public function testVersionWithSlash(): void
+    {
+        $dist = new Dist('repo', 'package', 'dev-master/feature', '123456', 'zip');
+
+        self::assertEquals(md5('dev-master/feature'), $dist->version());
+    }
+}