Skip to content

Commit

Permalink
Don't mirroring a package if it exists at the same name in canonical …
Browse files Browse the repository at this point in the history
…private repository
  • Loading branch information
vtsykun committed Feb 20, 2023
1 parent aed3480 commit 8775f0a
Show file tree
Hide file tree
Showing 17 changed files with 436 additions and 40 deletions.
4 changes: 4 additions & 0 deletions config/services.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ parameters:
package_name_regex_v2: '[A-Za-z0-9_.-]+/[A-Za-z0-9_.~-]+'
package_name_regex_v1: '[A-Za-z0-9_.-]+/[A-Za-z0-9_.$-]+' # package$hash

mirror_metadata_regex: '[A-Za-z0-9_.-]+/[@A-Za-z0-9_.-]+'
mirror_metadata_regex_v2: '[A-Za-z0-9_.-]+/[@A-Za-z0-9_.~-]+'
mirror_metadata_regex_v1: '[A-Za-z0-9_.-]+/[@A-Za-z0-9_.$-]+'

services:
# default configuration for services in *this* file
_defaults:
Expand Down
3 changes: 0 additions & 3 deletions public/packeton/js/proxies.js
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,6 @@
let updateBtn = $('.update.action').find('.btn');
updateBtn.on('click', (e) => {
e.preventDefault();
$('#json-model').modal({show: true});

return;
let data = {};
let options = ajaxFormData(e.target, data);
if (data['force']) {
Expand Down
132 changes: 132 additions & 0 deletions src/Composer/IO/BufferIO.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
<?php

declare(strict_types=1);

namespace Packeton\Composer\IO;

use Composer\IO\ConsoleIO;
use Composer\IO\IOInterface;
use Composer\Pcre\Preg;
use Symfony\Component\Console\Formatter\OutputFormatterInterface;
use Symfony\Component\Console\Helper\HelperSet;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Helper\QuestionHelper;
use Symfony\Component\Console\Input\StringInput;
use Symfony\Component\Console\Output\BufferedOutput;
use Symfony\Component\Console\Output\NullOutput;
use Symfony\Component\Console\Output\StreamOutput;

/**
* To show more logs in UI, allow to out limit count of debugs logs.
*/
class BufferIO extends ConsoleIO
{
protected $output;

protected $verbosityMatrixCapacity = 3;

protected $verbosityMatrix = [
IOInterface::VERBOSE => 3,
IOInterface::VERY_VERBOSE => 2,
IOInterface::DEBUG => 2,
];

public function __construct(string $input = '', int $verbosity = StreamOutput::VERBOSITY_NORMAL, ?OutputFormatterInterface $formatter = null)
{
$input = new StringInput($input);
$input->setInteractive(false);

$this->output = $output = new BufferedOutput($verbosity, $formatter && $formatter->isDecorated(), $formatter);

parent::__construct($input, $output, new HelperSet([
new QuestionHelper(),
]));
}

/**
* @return string output
*/
public function getOutput(): string
{
$output = $this->output->fetch();

$output = Preg::replaceCallback("{(?<=^|\n|\x08)(.+?)(\x08+)}", static function ($matches): string {
assert(is_string($matches[1]));
assert(is_string($matches[2]));
$pre = strip_tags($matches[1]);

if (strlen($pre) === strlen($matches[2])) {
return '';
}

// TODO reverse parse the string, skipping span tags and \033\[([0-9;]+)m(.*?)\033\[0m style blobs
return rtrim($matches[1])."\n";
}, $output);

return $output;
}

public function getProgressBar(int $max = 0)
{
return new ProgressBar(new NullOutput(), $max);
}

public function overwrite($messages, bool $newline = true, ?int $size = null, int $verbosity = self::NORMAL)
{
$messages = strip_tags($messages, ['error', 'warning', 'info']);
parent::overwriteError($messages, $newline, $size, $this->dynamicVerbosity($verbosity));
}

public function overwriteError($messages, bool $newline = true, ?int $size = null, int $verbosity = self::NORMAL)
{
$messages = strip_tags($messages, ['error', 'warning', 'info']);
parent::overwriteError($messages, $newline, $size, $this->dynamicVerbosity($verbosity));
}

public function write($messages, bool $newline = true, int $verbosity = self::NORMAL)
{
$messages = strip_tags($messages, ['error', 'warning', 'info']);
parent::write($messages, $newline, $this->dynamicVerbosity($verbosity));
}

public function writeError($messages, bool $newline = true, int $verbosity = self::NORMAL)
{
$messages = strip_tags($messages, ['error', 'warning', 'info']);
parent::writeError($messages, $newline, $this->dynamicVerbosity($verbosity));
}

public function writeRaw($messages, bool $newline = true, int $verbosity = self::NORMAL)
{
$messages = strip_tags($messages, ['error', 'warning', 'info']);
parent::writeRaw($messages, $newline, $this->dynamicVerbosity($verbosity));
}

public function writeErrorRaw($messages, bool $newline = true, int $verbosity = self::NORMAL)
{
$messages = strip_tags($messages, ['error', 'warning', 'info']);
parent::writeErrorRaw($messages, $newline, $this->dynamicVerbosity($verbosity));
}

public function log($level, $message, array $context = []): void
{
parent::log($level, \htmlspecialchars((string) $message), $context);
}

protected function dynamicVerbosity($verbosity)
{
if ($verbosity === self::NORMAL && $this->verbosityMatrixCapacity > 0) {
foreach ($this->verbosityMatrix as $verb => $value) {
if ($value <= 0) {
$this->verbosityMatrix[$verb] = 2;
$this->verbosityMatrixCapacity--;
}
}
}

if (isset($this->verbosityMatrix[$verbosity]) && $this->verbosityMatrix[$verbosity] > 0) {
return self::NORMAL;
}

return $verbosity;
}
}
6 changes: 3 additions & 3 deletions src/Controller/MirrorController.php
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ public function rootAction(Request $request, string $alias): Response
return $this->renderMetadata($metadata, $request, fn ($meta) => $this->metadataMerger->merge($meta, $api));
}

#[Route('/{alias}/p2/{package}.json', name: 'mirror_metadata_v2', requirements: ['package' => '%package_name_regex_v2%'], methods: ['GET'])]
#[Route('/{alias}/p2/{package}.json', name: 'mirror_metadata_v2', requirements: ['package' => '%mirror_metadata_regex_v2%'], methods: ['GET'])]
public function metadataV2Action(string $package, string $alias, Request $request): Response
{
$devStability = \str_ends_with($package, '~dev');
Expand All @@ -68,7 +68,7 @@ public function metadataV2Action(string $package, string $alias, Request $reques
return $this->renderMetadata($metadata, $request);
}

#[Route('/{alias}/pkg/{package}.json', name: 'mirror_metadata_v1', requirements: ['package' => '%package_name_regex_v1%'], methods: ['GET'])]
#[Route('/{alias}/pkg/{package}.json', name: 'mirror_metadata_v1', requirements: ['package' => '%mirror_metadata_regex_v1%'], methods: ['GET'])]
public function packageAction(string $package, string $alias, Request $request): Response
{
$metadata = $this->wrap404Error($alias, fn (PRI $repo) => $repo->findPackageMetadata($package));
Expand All @@ -79,7 +79,7 @@ public function packageAction(string $package, string $alias, Request $request):
#[Route(
'/{alias}/zipball/{package}/{version}/{ref}.{type}',
name: 'mirror_zipball',
requirements: ['package' => '%package_name_regex%'],
requirements: ['package' => '%mirror_metadata_regex%'],
methods: ['GET']
)]
public function zipball(string $alias, string $package, string $version, string $ref): Response
Expand Down
55 changes: 53 additions & 2 deletions src/Controller/ProxiesController.php
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,21 @@
namespace Packeton\Controller;

use Composer\Downloader\TransportException;
use Packeton\Composer\MetadataMinifier;
use Packeton\Form\Type\ProxySettingsType;
use Packeton\Mirror\Exception\MetadataNotFoundException;
use Packeton\Mirror\Model\ProxyInfoInterface;
use Packeton\Mirror\Model\ProxyOptions;
use Packeton\Mirror\ProxyRepositoryRegistry;
use Packeton\Mirror\RemoteProxyRepository;
use Packeton\Mirror\Service\ComposeProxyRegistry;
use Packeton\Mirror\Service\RemoteSyncProxiesFacade;
use Packeton\Mirror\Utils\MirrorPackagesValidate;
use Packeton\Mirror\Utils\MirrorTextareaParser;
use Packeton\Mirror\Utils\MirrorUIFormatter;
use Packeton\Model\PackageManager;
use Packeton\Service\JobScheduler;
use Packeton\Util\HtmlJsonHuman;
use Symfony\Bundle\FrameworkBundle\Controller\AbstractController;
use Symfony\Component\HttpFoundation\JsonResponse;
use Symfony\Component\HttpFoundation\Request;
Expand All @@ -26,8 +33,12 @@ class ProxiesController extends AbstractController
{
public function __construct(
private readonly ProxyRepositoryRegistry $proxyRepositoryRegistry,
private readonly ComposeProxyRegistry $composeProxyRegistry,
private readonly MirrorTextareaParser $textareaParser,
private readonly JobScheduler $jobScheduler,
private readonly MirrorPackagesValidate $mirrorValidate,
private readonly MetadataMinifier $metadataMinifier,
private readonly PackageManager $packageManager,
) {
}

Expand Down Expand Up @@ -80,6 +91,44 @@ public function settings(Request $request, string $alias)
]);
}

#[Route(
'/{alias}/metadata/{package}',
name: 'proxy_package_meta',
requirements: ['package' => '.+'],
methods: ["POST", "GET"]
)]
public function metadata(HtmlJsonHuman $jsonHuman, string $alias, string $package)
{
try {
$repo = $this->composeProxyRegistry->createRepository($alias);
$meta = $repo->findPackageMetadata($package);
} catch (MetadataNotFoundException $e) {
return new JsonResponse(['error' => $e->getMessage()], 404);
}

$json = $meta->decodeJson();
$metadata = $this->metadataMinifier->minify($json)['packages'][$package] ?? [];

return new Response($jsonHuman->buildToHtml($metadata));
}

#[Route('/{alias}/update', name: 'proxy_update', methods: ["PUT"])]
public function updateAction(Request $request, string $alias)
{
$repo = $this->getRemoteRepository($alias);
$data = $this->jsonRequest($request);
$flags = ($data['force'] ?? false) ? RemoteSyncProxiesFacade::FULL_RESET : 0;
$flags |= RemoteSyncProxiesFacade::UI_RESET;

$job = $this->jobScheduler->publish(
'sync:mirrors',
['mirror' => $alias, 'flags' => $flags],
$repo->getConfig()->reference()
);

return new JsonResponse(['job' => $job->getId()], 201);
}

#[Route('/{alias}/mark-enabled', name: 'proxy_mark_mass', methods: ["POST"])]
public function markMass(Request $request, string $alias)
{
Expand All @@ -88,9 +137,10 @@ public function markMass(Request $request, string $alias)

$packages = $this->textareaParser->parser($data['packages'] ?? null);
$action = $data['action'] ?? 'approve';
$pm = $repo->getPackageManager();

try {
$result = $this->mirrorValidate->checkPackages($repo, $packages);
$result = $this->mirrorValidate->checkPackages($repo, $packages, $pm->getEnabled());
} catch (TransportException $e) {
return new JsonResponse(['error' => $e->getMessage()], 400);
}
Expand Down Expand Up @@ -154,8 +204,9 @@ protected function getProxyData(RemoteProxyRepository $repo): array
$repoUrl = $this->generateUrl('mirror_index', ['alias' => $config->getAlias()], UrlGeneratorInterface::ABSOLUTE_URL);

$rpm = $repo->getPackageManager();
$privatePackages = $this->packageManager->getPackageNames();

$packages = MirrorUIFormatter::getGridPackagesData($rpm->getApproved(), $rpm->getEnabled());
$packages = MirrorUIFormatter::getGridPackagesData($rpm->getApproved(), $rpm->getEnabled(), $privatePackages);

return [
'repoUrl' => $repoUrl,
Expand Down
6 changes: 6 additions & 0 deletions src/Mirror/Decorator/ProxyRepositoryACLDecorator.php
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ public function __construct(
protected ?RemoteProxyRepository $remote = null,
protected array $availablePackages = [],
protected array $availablePackagePatterns = [],
protected array $excludedPackages = [],
) {
}

Expand Down Expand Up @@ -78,6 +79,11 @@ public function findPackageMetadata(string $nameOrUri): JsonMetadata
}

return $metadata;
} else if (\in_array($package, $this->excludedPackages)) {
throw new ApproveRestrictException(
"The package '$package' has been already registered in the your private repository. " .
"3-rd party mirrored packages always is canonical, so do not allowed use it if your already add private package by same name"
);
}

if ($this->availablePackages && !\in_array($package, $this->availablePackages, true)) {
Expand Down
24 changes: 16 additions & 8 deletions src/Mirror/Model/HttpMetadataTrait.php
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@ trait HttpMetadataTrait
/**
* @throws TransportException
*/
private function requestMetadataVia2(HttpDownloader $downloader, iterable $packages, string $url, callable $onFulfilled, callable $reject = null): void
private function requestMetadataVia2(HttpDownloader $downloader, iterable $packages, string $url, callable $onFulfilled, callable $onReject = null): void
{
$queue = new \ArrayObject();
$loop = new SignalLoop($downloader, $this->signal);

$promise = [];

$reject ??= static function ($e) {
$onReject ??= static function ($e, $package) {
if ($e instanceof TransportException && $e->getStatusCode() === 404) {
return false;
}
Expand All @@ -43,19 +43,23 @@ private function requestMetadataVia2(HttpDownloader $downloader, iterable $packa
}
};

$reject = function ($e) use ($package, $onReject) {
return $onReject($e, $package);
};

$promise[] = $downloader->add(\str_replace('%package%', $package, $url))->then($requester, $reject);
$promise[] = $downloader->add(\str_replace('%package%', $package . '~dev', $url))->then($requester, $reject);
}

$loop->wait($promise);
}

private function requestMetadataVia1(HttpDownloader $downloader, iterable $packages, string $url, callable $onFulfilled, callable $reject = null, iterable $providersGenerator = []): void
private function requestMetadataVia1(HttpDownloader $downloader, iterable $packages, string $url, callable $onFulfilled, callable $onReject = null, iterable $providersGenerator = []): void
{
$resolvedPackages = $promise = [];
$loop = new SignalLoop($downloader, $this->signal);

$reject ??= static function ($e) {
$onReject ??= static function ($e, $package) {
if ($e instanceof TransportException && $e->getStatusCode() === 404) {
return false;
}
Expand Down Expand Up @@ -83,10 +87,14 @@ private function requestMetadataVia1(HttpDownloader $downloader, iterable $packa
}

foreach ($resolvedPackages as $package => [$packageUrl, $hash]) {
$requester = function (Response $response) use ($package, $onFulfilled, $hash, $asArray) {
$response->getBody() ? $onFulfilled($package, $asArray ? $response->decodeJson() : $response, $hash) : null;
};
$promise[] = $downloader->add($packageUrl)->then($requester, $reject);
$promise[] = $downloader->add($packageUrl)->then(
function (Response $response) use ($package, $onFulfilled, $hash, $asArray) {
$response->getBody() ? $onFulfilled($package, $asArray ? $response->decodeJson() : $response, $hash) : null;
},
function ($e) use ($package, $onReject) {
return $onReject($e, $package);
}
);
}

$loop->wait($promise);
Expand Down
Loading

0 comments on commit 8775f0a

Please sign in to comment.