Skip to content

Commit

Permalink
Merge pull request osTicket#1661 from greezybacon/feature/file-import…
Browse files Browse the repository at this point in the history 
…-export

file: Add import/export support

Reviewed-By: Peter Rotich <peter@osticket.com>
  • Loading branch information
@protich
protich committed Mar 10, 2015
2 parents ff9f6e9 + 35dcc29 commit f9997d8
Show file tree
Hide file tree
Showing 3 changed files with 237 additions and 21 deletions.
4 changes: 2 additions & 2 deletions include/class.orm.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,9 @@ function save($refetch=false) {
foreach ($this->dirty as $field=>$old) {
if ($this->__new__ or !in_array($field, $pk)) {
if (@get_class($this->get($field)) == 'SqlFunction')
$fields[] = $field.' = '.$this->get($field)->toSql();
$fields[] = "`$field` = ".$this->get($field)->toSql();
else
$fields[] = $field.' = '.db_input($this->get($field));
$fields[] = "`$field` = ".db_input($this->get($field));
}
}
$sql .= ' SET '.implode(', ', $fields);
Expand Down
7 changes: 3 additions & 4 deletions setup/cli/modules/class.module.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,11 +92,10 @@ function toString() {
class OutputStream {
var $stream;

function OutputStream() {
call_user_func_array(array($this, '__construct'), func_get_args());
}
function __construct($stream) {
$this->stream = fopen($stream, 'w');
if (!($this->stream = fopen($stream, 'w')))
throw new Exception(sprintf('%s: Cannot open for writing',
$stream));
}

function write($what) {
Expand Down
247 changes: 232 additions & 15 deletions setup/cli/modules/file.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ class FileManager extends Module {
'list' => 'List files matching criteria',
'export' => 'Export files from the system',
'import' => 'Load files exported via `export`',
'zip' => 'Create a zip file of the matching files',
'dump' => 'Dump file content to stdout',
'load' => 'Load file contents from stdin',
'migrate' => 'Migrate a file to another backend',
Expand Down Expand Up @@ -176,11 +177,224 @@ function run($args, $options) {
$this->stdout->write("Migrated $count files\n");
break;

/**
* export
*
* Export file contents to a stream file. The format of the stream
* will be a continuous stream of file information in the following
* format:
*
* AFIL<meta-length><data-length><meta><data>EOF\x1c
*
* Where
* A is the version code of the export
* "FIL" is the literal text 'FIL'
* meta-length is 'V' packed header length (bytes)
* data-length is 'V' packed data length (bytes)
* meta is the %file record, php serialized
* data is the raw content of the file
* "EOF" is the literal text 'EOF'
* \x1c is an ASCII 0x1c byte (file separator)
*
* Options:
* --file File to which to direct the stream output, default
* is stdout
*/
case 'export':
// Create a temporary ZIP file
$files = FileModel::objects();
$this->_applyCriteria($options, $files);

if (!$options['file'] || $options['file'] == '-')
$options['file'] = 'php://stdout';

if (!($stream = fopen($options['file'], 'wb')))
$this->fail($options['file'].': Unable to open file for export stream');

foreach ($files as $m) {
$f = AttachmentFile::lookup($m->id);
if ($options['verbose'])
$this->stderr->write($m->name."\n");

// TODO: Log %attachment and %ticket_attachment entries
$info = array('file' => $f->getInfo());
$header = serialize($info);
fwrite($stream, 'AFIL'.pack('VV', strlen($header), $f->getSize()));
fwrite($stream, $header);
$FS = $f->open();
while ($block = $FS->read())
fwrite($stream, $block);
fwrite($stream, "EOF\x1c");
}
fclose($stream);
break;

/**
* import
*
* Import a collection of file contents exported by the `export`.
* See the export function above for details about the stream
* format.
*
* Options:
* --file File from which to read the export stream, default
* is stdin
* --to Backend to receive the contents (@see `backends`)
* --verbose Show file names while importing
*/
case 'import':
if (!$options['file'] || $options['file'] == '-')
$options['file'] = 'php://stdin';

if (!($stream = fopen($options['file'], 'rb')))
$this->fail($options['file'].': Unable to open import stream');

while (true) {
// Read the file header
// struct file_data_header {
// char[4] marker; // Four chars, 'AFIL'
// int lenMeta;
// int lenData;
// };
if (!($header = fread($stream, 12)))
break; // EOF

list(, $mark, $hlen, $dlen) = unpack('V3', $header);

// AFIL written as little-endian 4-byte int is 0x4c4946xx (LIFA),
// where 'A' is the version code of the export
$version = $mark & 0xff;
if (($mark >> 8) != 0x4c4946)
$this->fail('Bad file record');

// Read the header
$header = fread($stream, $hlen);
if (strlen($header) != $hlen)
$this->fail('Short read getting header info');

$header = unserialize($header);
if (!$header)
$this->fail('Unable to decipher file header');

// Find or create the file record
$finfo = $header['file'];
// TODO: Consider the $version code
$f = AttachmentFile::lookup($finfo['id']);
if ($f) {
// Verify file information
if ($f->getSize() != $finfo['size']
|| $f->getSignature() != $finfo['signature']
) {
$this->fail(sprintf(
'%s: File data does not match existing file record',
$finfo['name']
));
}
// Drop existing file contents, if any
try {
if ($bk = $f->open())
$bk->unlink();
}
catch (Exception $e) {}
}
// Create a new file
else {
$fm = FileModel::create($finfo);
if (!$fm->save() || !($f = AttachmentFile::lookup($fm->id))) {
$this->fail(sprintf(
'%s: Unable to create new file record',
$finfo['name']));
}
}

// Determine the backend to recieve the file contents
if ($options['to']) {
$bk = FileStorageBackend::lookup($options['to'], $f);
}
// Use the system default
else {
$bk = AttachmentFile::getBackendForFile($f);
}

if ($options['verbose'])
$this->stdout->write('Importing '.$f->getName()."\n");

// Write file contents to the backend
$md5 = hash_init('md5');
$sha1 = hash_init('sha1');
$written = 0;

// Handle exceptions by dropping imported file contents and
// then returning the error to the error output stream.
try {
while ($dlen > 0) {
$read_size = min($dlen, $bk->getBlockSize());
$contents = '';
// reading from the stream will likely return an amount of
// data different from the backend requested block size. Loop
// until $read_size bytes are recieved.
while ($read_size > 0 && ($block = fread($stream, $read_size))) {
$contents .= $block;
$read_size -= strlen($block);
}
if ($read_size != 0) {
// short read
throw new Exception(sprintf(
'%s: Some contents are missing from the stream',
$f->getName()
));
}
// Calculate MD5 and SHA1 hashes of the file to verify
// contents after successfully written to backend
if (!$bk->write($contents))
throw new Exception(
'Unable to send file contents to backend');
hash_update($md5, $contents);
hash_update($sha1, $contents);
$dlen -= strlen($contents);
$written += strlen($contents);
}
// Some backends cannot handle flush() without a
// corresponding write() call.
if ($written && !$bk->flush())
throw new Exception(
'Unable to commit file contents to backend');

// Check the signature hash
if ($finfo['signature']) {
$md5 = base64_encode(hash_final($md5, true));
$sha1 = base64_encode(hash_final($sha1, true));
$sig = str_replace(
array('=','+','/'),
array('','-','_'),
substr($sha1, 0, 16) . substr($md5, 0, 16));
if ($sig != $finfo['signature']) {
throw new Exception(sprintf(
'%s: Signature verification failed',
$f->getName()
));
}
}
} // end try
catch (Exception $ex) {
if ($bk) $bk->unlink();
$this->fail($ex->getMessage());
}

// Read file record footer
$footer = fread($stream, 4);
if (strlen($footer) != 4)
$this->fail('Unable to read file EOF marker');
list(, $footer) = unpack('N', $footer);
// Footer should be EOF\x1c as an int
if ($footer != 0x454f461c)
$this->fail('Incorrect file EOF marker');
}
break;

case 'zip':
// Create a temporary ZIP file
$files = FileModel::objects();
$this->_applyCriteria($options, $files);
if (!$options['file'])
$this->fail('Please specify zip file with `-f`');

Expand All @@ -189,30 +403,33 @@ function run($args, $options) {
ZipArchive::CREATE)))
$this->fail($reason.': Unable to create zip file');

$manifest = array();
foreach ($files as $m) {
$f = AttachmentFile::lookup($m->id);
$zip->addFromString($f->getId(), $f->getData());
$zip->setCommentName($f->getId(), $f->getName());
// TODO: Log %attachment and %ticket_attachment entries
$info = array('file' => $f->getInfo());
foreach ($m->tickets as $t)
$info['tickets'][] = $t->ht;

$manifest[$f->getId()] = $info;
if ($options['verbose'])
$this->stderr->write($m->name."\n");
$name = Format::encode(sprintf(
'%d-%s', $f->getId(), $f->getName()
), 'utf-8', 'cp437');
$zip->addFromString($name, $f->getData());
}
$zip->addFromString('MANIFEST', serialize($manifest));
$zip->close();
break;

case 'expunge':
// Create a temporary ZIP file
$files = FileModel::objects();
$this->_applyCriteria($options, $files);

foreach ($files as $f) {
$f->tickets->expunge();
$f->unlink() && $f->delete();
foreach ($files as $m) {
// Drop associated attachment links
$m->tickets->expunge();
$f = AttachmentFile::lookup($m->id);

// Drop file contents
if ($bk = $f->open())
$bk->unlink();

// Drop file record
$f->delete();
}
}
}
Expand Down

0 comments on commit f9997d8

Please sign in to comment.