Skip to content

Commit

Permalink
Merge pull request kaltura#6652 from kaltura/Mercury-13.11.0-PLAT-846…
Browse files Browse the repository at this point in the history 
…9-EnableEncryptionInParts

PLAT-8469: change encryption method and support encrypt of large files
  • Loading branch information
@david-winder-kaltura
david-winder-kaltura authored Jan 18, 2018
2 parents b96995d + f3771aa commit 168e404
Show file tree
Hide file tree
Showing 12 changed files with 113 additions and 57 deletions.
9 changes: 3 additions & 6 deletions alpha/apps/kaltura/lib/myEntryUtils.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -950,9 +950,7 @@ public static function resizeEntryImage( entry $entry, $version , $width , $heig

if ($isEncryptionNeeded)
{
$maxFileSize = kConf::get('max_file_size_for_encryption', 'local', FileSync::MAX_FILE_SIZE_FOR_ENCRYPTION);
if (filesize($finalThumbPath) < $maxFileSize)
$finalThumbPath = self::encryptThumb($finalThumbPath, $entry->getGeneralEncryptionKey(), $entry->getEncryptionIv());
$finalThumbPath = self::encryptThumb($finalThumbPath, $entry->getGeneralEncryptionKey(), $entry->getEncryptionIv());
}

return $finalThumbPath;
Expand All @@ -965,10 +963,9 @@ private static function isTempFile($filePath)

private static function encryptThumb($thumbPath, $key, $iv)
{
if (!kEncryptFileUtils::encryptFile($thumbPath, $key, $iv))
return $thumbPath;
$encryptedPath = kFileUtils::addEncryptToFileName($thumbPath);
kFile::moveFile($thumbPath, $encryptedPath);
if (!kEncryptFileUtils::encryptFile($thumbPath, $key, $iv, $encryptedPath))
return $thumbPath;
KalturaLog::debug("Data for entry should encrypted. Encrypted data at [$encryptedPath] with key [$key] and iv [$iv]");
return $encryptedPath;
}
Expand Down
6 changes: 3 additions & 3 deletions alpha/apps/kaltura/lib/renderers/kRendererDumpFile.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ class kRendererDumpFile implements kRendererBase

public $partnerId;

public function __construct($filePath, $mimeType, $xSendFileAllowed, $maxAge = 8640000, $limitFileSize = 0, $lastModified = null, $key = null, $iv = null)
public function __construct($filePath, $mimeType, $xSendFileAllowed, $maxAge = 8640000, $limitFileSize = 0, $lastModified = null, $key = null, $iv = null, $fileSize = null)
{
$this->filePath = $filePath;
$this->mimeType = $mimeType;
Expand All @@ -42,7 +42,7 @@ public function __construct($filePath, $mimeType, $xSendFileAllowed, $maxAge = 8
else
{
clearstatcache();
$this->fileSize = kEncryptFileUtils::fileSize($filePath, $key, $iv);
$this->fileSize = $fileSize ? $fileSize : kEncryptFileUtils::fileSize($filePath, $key, $iv);
$this->xSendFileAllowed = $xSendFileAllowed;
}

Expand Down Expand Up @@ -98,7 +98,7 @@ public function output()
else
{
if ($this->key)
echo kEncryptFileUtils::getEncryptedFileContent($this->filePath, $this->key, $this->iv, $rangeFrom, $rangeLength);
kEncryptFileUtils::dumpEncryptFilePart($this->filePath, $this->key, $this->iv, $rangeFrom, $rangeLength);
else
infraRequestUtils::dumpFilePart($this->filePath, $rangeFrom, $rangeLength);
}
Expand Down
2 changes: 1 addition & 1 deletion alpha/apps/kaltura/lib/storage/kDataCenterMgr.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ public static function serveFileToRemoteDataCenter ( $file_sync , $file_hash, $f
else
{
KalturaLog::log("Serving file from [".$resolvedPath."]");
kFileUtils::dumpFile( $resolvedPath , null, null, 0 ,$file_sync_resolved->getEncryptionKey(), $file_sync_resolved->getIv());
kFileUtils::dumpFile( $resolvedPath , null, null, 0 ,$file_sync_resolved->getEncryptionKey(), $file_sync_resolved->getIv(), $file_sync_resolved->getFileSize());
}

}
Expand Down
2 changes: 1 addition & 1 deletion alpha/apps/kaltura/lib/storage/kFileSyncUtils.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1631,7 +1631,7 @@ public static function dumpFileByFileSync( FileSync $fileSync)
$resolveFileSync = self::resolve($fileSync);
$path = $resolveFileSync->getFullPath();
KalturaLog::info("Resolve path [$path]");
kFileUtils::dumpFile($path, null, null, 0, $fileSync->getEncryptionKey(), $fileSync->getIv());
kFileUtils::dumpFile($path, null, null, 0, $fileSync->getEncryptionKey(), $fileSync->getIv(), $fileSync->getFileSize());
}

public static function dumpFileByFileSyncKey( FileSyncKey $key , $strict = false )
Expand Down
8 changes: 4 additions & 4 deletions alpha/apps/kaltura/lib/storage/kFileUtils.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public static function xSendFileAllowed($file_name)
return false;
}

public static function getDumpFileRenderer($filePath, $mimeType, $maxAge = null, $limitFileSize = 0, $lastModified = null, $key = null, $iv = null)
public static function getDumpFileRenderer($filePath, $mimeType, $maxAge = null, $limitFileSize = 0, $lastModified = null, $key = null, $iv = null, $fileSize = null)
{
self::closeDbConnections();

Expand All @@ -59,12 +59,12 @@ public static function getDumpFileRenderer($filePath, $mimeType, $maxAge = null,
if(! file_exists($filePath))
KExternalErrors::dieError(KExternalErrors::FILE_NOT_FOUND);

return new kRendererDumpFile($filePath, $mimeType, self::xSendFileAllowed($filePath), $maxAge, $limitFileSize, $lastModified, $key, $iv);
return new kRendererDumpFile($filePath, $mimeType, self::xSendFileAllowed($filePath), $maxAge, $limitFileSize, $lastModified, $key, $iv, $fileSize);
}

public static function dumpFile($file_name, $mime_type = null, $max_age = null, $limit_file_size = 0, $key = null, $iv = null)
public static function dumpFile($file_name, $mime_type = null, $max_age = null, $limit_file_size = 0, $key = null, $iv = null, $fileSize = null)
{
$renderer = self::getDumpFileRenderer($file_name, $mime_type, $max_age, $limit_file_size, null, $key, $iv);
$renderer = self::getDumpFileRenderer($file_name, $mime_type, $max_age, $limit_file_size, null, $key, $iv, $fileSize);

$renderer->output();

Expand Down
7 changes: 4 additions & 3 deletions alpha/apps/kaltura/modules/extwidget/actions/downloadAction.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ public function execute()
list ($fileSync,$local) = kFileSyncUtils::getReadyFileSyncForKey($syncKey, false, false);
if (!$fileSync)
KExternalErrors::dieError(KExternalErrors::FILE_NOT_FOUND);
/**@var $fileSync FileSync */
$filePath = $fileSync->getFullPath();

list($fileBaseName, $fileExt) = kAssetUtils::getFileName($entry, $flavorAsset);
Expand All @@ -143,7 +144,7 @@ public function execute()
//enable downloading file_name which inside the flavor asset directory
if(is_dir($filePath))
$filePath = $filePath.DIRECTORY_SEPARATOR.$fileName;
$this->dumpFile($filePath, $fileName, $preview, $fileSync->getEncryptionKey(), $fileSync->getIv());
$this->dumpFile($filePath, $fileName, $preview, $fileSync->getEncryptionKey(), $fileSync->getIv(), $fileSync->getFileSize());

KExternalErrors::dieGracefully(); // no view
}
Expand Down Expand Up @@ -174,7 +175,7 @@ private function getBestSyncKeyForEntry(entry $entry)
return $syncKey;
}

private function dumpFile($file_path, $file_name, $limit_file_size = 0, $key = null, $iv = null)
private function dumpFile($file_path, $file_name, $limit_file_size = 0, $key = null, $iv = null, $fileSize = null)
{
$file_name = str_replace("\n", ' ', $file_name);
$relocate = $this->getRequestParameter("relocate");
Expand Down Expand Up @@ -205,7 +206,7 @@ private function dumpFile($file_path, $file_name, $limit_file_size = 0, $key = n
header("Content-Disposition: attachment; filename=\"$file_name\"");

$mime_type = kFile::mimeType($file_path);
kFileUtils::dumpFile($file_path, $mime_type, null, $limit_file_size, $key, $iv);
kFileUtils::dumpFile($file_path, $mime_type, null, $limit_file_size, $key, $iv, $fileSize);
}
}

Expand Down
14 changes: 3 additions & 11 deletions alpha/lib/model/FileSync.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ class FileSync extends BaseFileSync implements IBaseObject
const FILE_SYNC_STATUS_READY = 2;
const FILE_SYNC_STATUS_DELETED = 3;
const FILE_SYNC_STATUS_PURGED = 4;

const MAX_FILE_SIZE_FOR_ENCRYPTION = 3000000; // as 3MB

private $statusMap = array (
self::FILE_SYNC_STATUS_ERROR => "Error",
Expand Down Expand Up @@ -106,11 +104,6 @@ public function shouldEncryptFile()
return false;
}

$maxFileSize = kConf::get('max_file_size_for_encryption', 'local', self::MAX_FILE_SIZE_FOR_ENCRYPTION);
if (filesize($this->getFullPath()) > $maxFileSize)
return false;


if ($this->getEncryptionKey())
{
KalturaLog::info("File of fileSyncId [$this->id] already has key and should not be encrypt again");
Expand All @@ -122,15 +115,14 @@ public function shouldEncryptFile()

public function setFileSizeFromPath ($filePath)
{

$fileSize = kEncryptFileUtils::fileSize($filePath, $this->getEncryptionKey(), $this->getIv());
$this->setFileSize($fileSize);
}

private function getClearTempPath()
{
$type = pathinfo($this->getFilePath(), PATHINFO_EXTENSION);
return sys_get_temp_dir(). "/". $this->getEncryptionKey() . ".$type";
return sys_get_temp_dir(). "/". $this->getEncryptionKey() . $this->getVersion() . ".$type";
}

public function getFullPath ()
Expand All @@ -140,11 +132,11 @@ public function getFullPath ()

public function createTempClear()
{
$plainData = $this->decrypt();
$realPath = realpath($this->getFullPath());
$tempPath = $this->getClearTempPath();
KalturaLog::info("Creating new file for syncId [$this->id] on [$tempPath]");
if (!file_exists($tempPath))
kFileBase::setFileContent( $tempPath, $plainData);
kEncryptFileUtils::decryptFile($realPath, $this->getEncryptionKey(), $this->getIv(), $tempPath);
return $tempPath;
}

Expand Down
5 changes: 3 additions & 2 deletions api_v3/lib/KalturaBaseService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,17 +309,18 @@ protected function getKsUniqueString()
* @param string $mimeType
* @param string $key
* @param string $iv
* @param int $fileSize
* @return kRendererDumpFile
*/
protected function dumpFile($filePath, $mimeType, $key = null, $iv = null)
protected function dumpFile($filePath, $mimeType, $key = null, $iv = null, $fileSize = null)
{
$maxAge = null;
if ($this->ks)
{
$maxAge = min(max($this->ks->valid_until - time(), 1), 8640000);
}

return kFileUtils::getDumpFileRenderer($filePath, $mimeType, $maxAge, 0, null, $key, $iv);
return kFileUtils::getDumpFileRenderer($filePath, $mimeType, $maxAge, 0, null, $key, $iv, $fileSize);
}

/**
Expand Down
8 changes: 4 additions & 4 deletions batch/batches/KBatchBase.class.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -645,11 +645,11 @@ function log($message)
public static function createTempClearFile($path, $key)
{
$iv = self::getIV();
$tempPath = sys_get_temp_dir(). "/clear_" . pathinfo($path, PATHINFO_BASENAME);
$tempPath = kEncryptFileUtils::getClearTempPath($path);
KalturaLog::info("Creating tempFile with Key is: [$key] iv: [$iv] for path [$path] at [$tempPath]");
$plainData = kEncryptFileUtils::getEncryptedFileContent($path, $key, $iv);
kFileBase::setFileContent($tempPath, $plainData);
return $tempPath;
if (kEncryptFileUtils::decryptFile($path, $key, $iv, $tempPath))
return $tempPath;
return null;
}

public static function getIV()
Expand Down
3 changes: 0 additions & 3 deletions configurations/base.ini
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,9 +124,6 @@ template_partner_id = 99
DropFolderListTTL = 300
ResourceReservationDuration = 5

max_file_size_for_encryption = 25000000


copy_partner_limit_entries = 20
copy_partner_limit_playlists = 10
copy_partner_limit_categories = 500
Expand Down
104 changes: 86 additions & 18 deletions infra/storage/kEncryptFileUtils.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,18 @@ class kEncryptFileUtils
{
//iv length should be 16
CONST ENCRYPT_METHOD = "AES-256-CBC";
const OPENSSL_RAW_DATA = 1;
const OPENSSL_RAW_DATA = 1; //can be removed once on PHP7
const ENCRYPT_INTERVAL = 3145728; // as 3MB = 1024 * 1024 * 3
const AES_BLOCK_SIZE = 16; //For IV extraction

public static function encryptData($plainText, $key, $iv)
{
$iv = substr($iv,0, openssl_cipher_iv_length(self::ENCRYPT_METHOD));
$encryptedData = openssl_encrypt($plainText, self::ENCRYPT_METHOD, $key, self::OPENSSL_RAW_DATA , $iv);
return base64_encode($encryptedData);
return openssl_encrypt($plainText, self::ENCRYPT_METHOD, $key, self::OPENSSL_RAW_DATA , $iv);
}

public static function decryptData($cipherText, $key, $iv)
{
$cipherText = base64_decode($cipherText);
$iv = substr($iv,0, openssl_cipher_iv_length(self::ENCRYPT_METHOD));
return openssl_decrypt($cipherText, self::ENCRYPT_METHOD, $key, self::OPENSSL_RAW_DATA, $iv);
}
Expand All @@ -38,40 +39,107 @@ public static function getEncryptedFileContent($fileName, $key = null, $iv = nul
return substr($plainData, $from_byte, $len);
}

public static function setEncryptedFileContent($fileName, $key, $iv, $content)
private static function doEncryptFile($srcFd, $key, $iv, $destFd)
{
$encryptedData = self::encryptData($content, $key, $iv);
kFileBase::setFileContent($fileName, $encryptedData);
$clear = fread($srcFd,self::ENCRYPT_INTERVAL);
$enc = self::encryptData($clear, $key, $iv);
$iv = substr($enc, -self::AES_BLOCK_SIZE);
fwrite($destFd, $enc);
return $iv;
}

public static function encryptFile($fileName, $key, $iv)
public static function encryptFile($srcFilePath, $key, $iv, $dstFilePath = null)
{
$data = kFileBase::getFileContent($fileName);
self::setEncryptedFileContent($fileName, $key, $iv, $data);
return true;
return self::wrapFileAccess('doEncryptFile', $srcFilePath, $key, $iv, $dstFilePath);
}

private static function doDecryptFile($srcFd, $key, $iv, $destFd)
{
$content = fread($srcFd, self::ENCRYPT_INTERVAL + self::AES_BLOCK_SIZE);
$clear = self::decryptData($content, $key, $iv);
fwrite($destFd, $clear);
return substr($content, -self::AES_BLOCK_SIZE);
}

public static function fileSize($filename, $key = null, $iv = null)
public static function decryptFile($srcFilePath, $key, $iv, $dstFilePath = null)
{
return self::wrapFileAccess('doDecryptFile', $srcFilePath, $key, $iv, $dstFilePath);
}

private static function wrapFileAccess($functionName, $srcFilePath, $key, $iv, $dstFilePath = null)
{
$fd1 = $fd2 = null;
try
{
$tempPath = self::getClearTempPath($srcFilePath);
$fd1 = fopen($srcFilePath, "rb");
$fd2 = fopen($tempPath, "w");
while (!feof($fd1))
{
$iv = call_user_func_array("self::$functionName", array($fd1, $key, $iv, $fd2));
}
fclose($fd1);
fclose($fd2);

if (!$dstFilePath)
$dstFilePath = $srcFilePath;
return rename($tempPath, $dstFilePath);
}
catch(Exception $e)
{
if ($fd1)
fclose($fd1);
if ($fd2)
fclose($fd2);
throw new Exception("Failed to [$functionName] for src path [$srcFilePath] and dest [$dstFilePath] because " . $e->getMessage());
}
}

public static function dumpEncryptFilePart($filePath, $key, $iv, $rangeFrom, $rangeLength)
{
$tempPath = self::getClearTempPath($filePath);
self::decryptFile($filePath, $key, $iv, $tempPath);
infraRequestUtils::dumpFilePart($tempPath, $rangeFrom, $rangeLength);
unlink($tempPath);
return;
}

public static function fileSize($filePath, $key = null, $iv = null)
{
$size = kFileBase::fileSize($filePath);
if (!$key)
return kFileBase::fileSize($filename);
$data = self::getEncryptedFileContent($filename, $key, $iv, 0, -1);
return strlen($data);
return $size;

if ($size < self::ENCRYPT_INTERVAL)
return strlen(self::getEncryptedFileContent($filePath, $key, $iv, 0, -1));

$tempPath = self::getClearTempPath($filePath);
self::decryptFile($filePath, $key, $iv, $tempPath);
$size = kFileBase::fileSize($tempPath);
unlink($tempPath);
return $size;
}

public static function encryptFolder($dirName, $key, $iv)
{
$filesPaths = kFile::dirList($dirName);
$done = true;
foreach ($filesPaths as $filePath)
self::encryptFile($filePath, $key, $iv);
$done &= self::encryptFile($filePath, $key, $iv);
return $done;
}

public static function encrypt($path, $key, $iv)
{
if (is_file($path))
self::encryptFile($path, $key, $iv);
return self::encryptFile($path, $key, $iv);
else if (is_dir($path))
self::encryptFolder($path, $key, $iv);
return self::encryptFolder($path, $key, $iv);
}

public static function getClearTempPath($path)
{
return sys_get_temp_dir(). "/clear_" . pathinfo($path, PATHINFO_BASENAME);
}


Expand Down
2 changes: 1 addition & 1 deletion plugins/content/caption/base/services/CaptionAssetService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ protected function attachFile(CaptionAsset $captionAsset, $fullPath, $copyOnly =

$captionAsset->setWidth($width);
$captionAsset->setHeight($height);
$captionAsset->setSize(kEncryptFileUtils::fileSize($finalPath, $fileSync->getEncryptionKey(), $fileSync->getIv()));
$captionAsset->setSize($fileSync->getFileSize());

$captionAsset->setStatus(CaptionAsset::ASSET_STATUS_READY);
$captionAsset->save();
Expand Down

0 comments on commit 168e404

Please sign in to comment.