123
This commit is contained in:
romkazvo
490
CrySystem/ZipDirCacheRW.cpp
Normal file
490
CrySystem/ZipDirCacheRW.cpp
Normal file
@@ -0,0 +1,490 @@
|
||||
#include "stdafx.h"
|
||||
#include <smartptr.h>
|
||||
#include "MTSafeAllocator.h"
|
||||
#include "ZipFileFormat.h"
|
||||
#include "ZipDirStructures.h"
|
||||
#include "ZipDirTree.h"
|
||||
#include "ZipDirList.h"
|
||||
#include "ZipDirCache.h"
|
||||
#include "ZipDirCacheRW.h"
|
||||
#include "ZipDirCacheFactory.h"
|
||||
#include "ZipDirFindRW.h"
|
||||
|
||||
|
||||
// declaration of Z_OK for ZipRawDecompress
|
||||
#include "zlib/zlib.h"
|
||||
|
||||
using namespace ZipFile;
|
||||
|
||||
|
||||
void ZipDir::CacheRW::Close()
|
||||
{
|
||||
if (m_pFile)
|
||||
{
|
||||
if (!(m_nFlags & FLAGS_READ_ONLY))
|
||||
{
|
||||
if ((m_nFlags & FLAGS_UNCOMPACTED) && !(m_nFlags&FLAGS_DONT_COMPACT))
|
||||
{
|
||||
if (!RelinkZip())
|
||||
WriteCDR();
|
||||
}
|
||||
else
|
||||
if (m_nFlags & FLAGS_CDR_DIRTY)
|
||||
WriteCDR();
|
||||
}
|
||||
|
||||
fclose (m_pFile);
|
||||
m_pFile = NULL;
|
||||
}
|
||||
m_pHeap = NULL;
|
||||
m_treeDir.Clear();
|
||||
}
|
||||
|
||||
|
||||
// Adds a new file to the zip or update an existing one
|
||||
// adds a directory (creates several nested directories if needed)
|
||||
ZipDir::ErrorEnum ZipDir::CacheRW::UpdateFile (const char* szRelativePath, void* pUncompressed, unsigned nSize, unsigned nCompressionMethod, int nCompressionLevel)
|
||||
{
|
||||
SmartPtr pBufferDestroyer(m_pHeap);
|
||||
|
||||
// we'll need the compressed data
|
||||
void* pCompressed;
|
||||
unsigned long nSizeCompressed;
|
||||
int nError;
|
||||
|
||||
switch (nCompressionMethod)
|
||||
{
|
||||
case METHOD_DEFLATE:
|
||||
// allocate memory for compression. Min is nSize * 1.001 + 12
|
||||
nSizeCompressed = nSize + (nSize >> 3) + 32;
|
||||
pCompressed = m_pHeap->Alloc(nSizeCompressed, "ZipDir::CacheRW::UpdateFile");
|
||||
pBufferDestroyer.Attach(pCompressed);
|
||||
nError = ZipRawCompress(m_pHeap, pUncompressed, &nSizeCompressed, pCompressed, nSize, nCompressionLevel);
|
||||
if (Z_OK != nError)
|
||||
return ZD_ERROR_ZLIB_FAILED;
|
||||
break;
|
||||
|
||||
case METHOD_STORE:
|
||||
pCompressed = pUncompressed;
|
||||
nSizeCompressed = nSize;
|
||||
break;
|
||||
|
||||
default:
|
||||
return ZD_ERROR_UNSUPPORTED;
|
||||
}
|
||||
|
||||
// create or find the file entry.. this object will rollback (delete the object
|
||||
// if the operation fails) if needed.
|
||||
FileEntryTransactionAdd pFileEntry(this, szRelativePath);
|
||||
|
||||
if (!pFileEntry)
|
||||
return ZD_ERROR_INVALID_PATH;
|
||||
|
||||
pFileEntry->OnNewFileData (pUncompressed, nSize, nSizeCompressed, nCompressionMethod);
|
||||
// since we changed the time, we'll have to update CDR
|
||||
m_nFlags |= FLAGS_CDR_DIRTY;
|
||||
|
||||
// the new CDR position, if the operation completes successfully
|
||||
unsigned lNewCDROffset = m_lCDROffset;
|
||||
|
||||
if (pFileEntry->IsInitialized())
|
||||
{
|
||||
// this file entry is already allocated in CDR
|
||||
|
||||
// check if the new compressed data fits into the old place
|
||||
unsigned nFreeSpace = pFileEntry->nEOFOffset - pFileEntry->nFileHeaderOffset - (unsigned)sizeof(ZipFile::LocalFileHeader) - (unsigned)strlen(szRelativePath);
|
||||
|
||||
if (nFreeSpace != nSizeCompressed)
|
||||
m_nFlags |= FLAGS_UNCOMPACTED;
|
||||
|
||||
if (nFreeSpace >= nSizeCompressed)
|
||||
{
|
||||
// and we can just override the compressed data in the file
|
||||
ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, szRelativePath);
|
||||
if (e != ZD_ERROR_SUCCESS)
|
||||
return e;
|
||||
}
|
||||
else
|
||||
{
|
||||
// we need to write the file anew - in place of current CDR
|
||||
pFileEntry->nFileHeaderOffset = m_lCDROffset;
|
||||
ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, szRelativePath);
|
||||
lNewCDROffset = pFileEntry->nEOFOffset;
|
||||
if (e != ZD_ERROR_SUCCESS)
|
||||
return e;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
pFileEntry->nFileHeaderOffset = m_lCDROffset;
|
||||
ErrorEnum e = WriteLocalHeader(m_pFile, pFileEntry, szRelativePath);
|
||||
if (e != ZD_ERROR_SUCCESS)
|
||||
return e;
|
||||
|
||||
lNewCDROffset = pFileEntry->nFileDataOffset + nSizeCompressed;
|
||||
|
||||
m_nFlags |= FLAGS_CDR_DIRTY;
|
||||
}
|
||||
|
||||
// now we have the fresh local header and data offset
|
||||
if (fseek (m_pFile, pFileEntry->nFileDataOffset, SEEK_SET)!=0
|
||||
||fwrite (pCompressed, nSizeCompressed, 1, m_pFile) != 1)
|
||||
{
|
||||
// we need to rollback the transaction: file write failed in the middle, the old
|
||||
// data is unrecoverably damaged, and we need to destroy this file entry
|
||||
// the CDR is already marked dirty
|
||||
return ZD_ERROR_IO_FAILED;
|
||||
}
|
||||
|
||||
// since we wrote the file successfully, update the new CDR position
|
||||
m_lCDROffset = lNewCDROffset;
|
||||
pFileEntry.Commit();
|
||||
|
||||
return ZD_ERROR_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
// deletes the file from the archive
|
||||
ZipDir::ErrorEnum ZipDir::CacheRW::RemoveFile (const char* szRelativePath)
|
||||
{
|
||||
// find the last slash in the path
|
||||
const char* pSlash = max(strrchr(szRelativePath, '/'), strrchr(szRelativePath, '\\'));
|
||||
|
||||
const char* pFileName; // the name of the file to delete
|
||||
|
||||
FileEntryTree* pDir; // the dir from which the subdir will be deleted
|
||||
|
||||
if (pSlash)
|
||||
{
|
||||
FindDirRW fd (GetRoot());
|
||||
// the directory to remove
|
||||
pDir = fd.FindExact(string (szRelativePath, pSlash-szRelativePath).c_str());
|
||||
if (!pDir)
|
||||
return ZD_ERROR_DIR_NOT_FOUND;// there is no such directory
|
||||
pFileName = pSlash+1;
|
||||
}
|
||||
else
|
||||
{
|
||||
pDir = GetRoot();
|
||||
pFileName = szRelativePath;
|
||||
}
|
||||
|
||||
ErrorEnum e = pDir->RemoveFile (pFileName);
|
||||
if (e == ZD_ERROR_SUCCESS)
|
||||
m_nFlags |= FLAGS_UNCOMPACTED|FLAGS_CDR_DIRTY;
|
||||
return e;
|
||||
}
|
||||
|
||||
|
||||
// deletes the directory, with all its descendants (files and subdirs)
|
||||
ZipDir::ErrorEnum ZipDir::CacheRW::RemoveDir (const char* szRelativePath)
|
||||
{
|
||||
// find the last slash in the path
|
||||
const char* pSlash = max(strrchr(szRelativePath, '/'), strrchr(szRelativePath, '\\'));
|
||||
|
||||
const char* pDirName; // the name of the dir to delete
|
||||
|
||||
FileEntryTree* pDir; // the dir from which the subdir will be deleted
|
||||
|
||||
if (pSlash)
|
||||
{
|
||||
FindDirRW fd (GetRoot());
|
||||
// the directory to remove
|
||||
pDir = fd.FindExact(string (szRelativePath, pSlash-szRelativePath).c_str());
|
||||
if (!pDir)
|
||||
return ZD_ERROR_DIR_NOT_FOUND;// there is no such directory
|
||||
pDirName = pSlash+1;
|
||||
}
|
||||
else
|
||||
{
|
||||
pDir = GetRoot();
|
||||
pDirName = szRelativePath;
|
||||
}
|
||||
|
||||
ErrorEnum e = pDir->RemoveDir (pDirName);
|
||||
if (e == ZD_ERROR_SUCCESS)
|
||||
m_nFlags |= FLAGS_UNCOMPACTED|FLAGS_CDR_DIRTY;
|
||||
return e;
|
||||
}
|
||||
|
||||
// deletes all files and directories in this archive
|
||||
ZipDir::ErrorEnum ZipDir::CacheRW::RemoveAll()
|
||||
{
|
||||
ErrorEnum e = m_treeDir.RemoveAll();
|
||||
if (e == ZD_ERROR_SUCCESS)
|
||||
m_nFlags |= FLAGS_UNCOMPACTED|FLAGS_CDR_DIRTY;
|
||||
return e;
|
||||
}
|
||||
|
||||
ZipDir::ErrorEnum ZipDir::CacheRW::ReadFile (FileEntry* pFileEntry, void* pCompressed, void* pUncompressed)
|
||||
{
|
||||
if (!pFileEntry)
|
||||
return ZD_ERROR_INVALID_CALL;
|
||||
|
||||
if (pFileEntry->desc.lSizeUncompressed == 0)
|
||||
{
|
||||
assert (pFileEntry->desc.lSizeCompressed == 0);
|
||||
return ZD_ERROR_SUCCESS;
|
||||
}
|
||||
|
||||
assert (pFileEntry->desc.lSizeCompressed > 0);
|
||||
|
||||
ErrorEnum nError = Refresh(pFileEntry);
|
||||
if (nError != ZD_ERROR_SUCCESS)
|
||||
return nError;
|
||||
|
||||
if (fseek (m_pFile, pFileEntry->nFileDataOffset, SEEK_SET))
|
||||
return ZD_ERROR_IO_FAILED;
|
||||
|
||||
SmartPtr pBufferDestroyer(m_pHeap);
|
||||
|
||||
void* pBuffer = pCompressed; // the buffer where the compressed data will go
|
||||
|
||||
if (pFileEntry->nMethod == 0 && pUncompressed)
|
||||
{
|
||||
// we can directly read into the uncompress buffer
|
||||
pBuffer = pUncompressed;
|
||||
}
|
||||
|
||||
if (!pBuffer)
|
||||
{
|
||||
if (!pUncompressed)
|
||||
// what's the sense of it - no buffers at all?
|
||||
return ZD_ERROR_INVALID_CALL;
|
||||
|
||||
pBuffer = m_pHeap->Alloc(pFileEntry->desc.lSizeCompressed, "ZipDir::Cache::ReadFile");
|
||||
pBufferDestroyer.Attach(pBuffer); // we want it auto-freed once we return
|
||||
}
|
||||
|
||||
if (fread (pBuffer, pFileEntry->desc.lSizeCompressed, 1, m_pFile) != 1)
|
||||
return ZD_ERROR_IO_FAILED;
|
||||
|
||||
// if there's a buffer for uncompressed data, uncompress it to that buffer
|
||||
if (pUncompressed)
|
||||
{
|
||||
if (pFileEntry->nMethod == 0)
|
||||
{
|
||||
assert (pBuffer == pUncompressed);
|
||||
//assert (pFileEntry->nSizeCompressed == pFileEntry->nSizeUncompressed);
|
||||
//memcpy (pUncompressed, pBuffer, pFileEntry->nSizeCompressed);
|
||||
}
|
||||
else
|
||||
{
|
||||
unsigned long nSizeUncompressed = pFileEntry->desc.lSizeUncompressed;
|
||||
if (Z_OK != ZipRawUncompress(m_pHeap, pUncompressed, &nSizeUncompressed, pBuffer, pFileEntry->desc.lSizeCompressed))
|
||||
return ZD_ERROR_CORRUPTED_DATA;
|
||||
}
|
||||
}
|
||||
|
||||
return ZD_ERROR_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////
|
||||
// finds the file by exact path
|
||||
ZipDir::FileEntry* ZipDir::CacheRW::FindFile (const char* szPath, bool bFullInfo)
|
||||
{
|
||||
if (!this)
|
||||
return NULL;
|
||||
ZipDir::FindFileRW fd (GetRoot());
|
||||
if (!fd.FindExact(szPath))
|
||||
{
|
||||
assert (!fd.GetFileEntry());
|
||||
return NULL;
|
||||
}
|
||||
assert (fd.GetFileEntry());
|
||||
return fd.GetFileEntry();
|
||||
}
|
||||
|
||||
// returns the size of memory occupied by the instance referred to by this cache
|
||||
size_t ZipDir::CacheRW::GetSize()const
|
||||
{
|
||||
return sizeof(*this) + m_strFilePath.capacity() + m_treeDir.GetSize() - sizeof(m_treeDir);
|
||||
}
|
||||
|
||||
// refreshes information about the given file entry into this file entry
|
||||
ZipDir::ErrorEnum ZipDir::CacheRW::Refresh (FileEntry* pFileEntry)
|
||||
{
|
||||
if (!pFileEntry)
|
||||
return ZD_ERROR_INVALID_CALL;
|
||||
|
||||
if (pFileEntry->nFileDataOffset != pFileEntry->INVALID_DATA_OFFSET)
|
||||
return ZD_ERROR_SUCCESS; // the data offset has been successfully read..
|
||||
|
||||
if (!this)
|
||||
return ZD_ERROR_INVALID_CALL; // from which cache is this file entry???
|
||||
|
||||
return ZipDir::Refresh(m_pFile, pFileEntry);
|
||||
}
|
||||
|
||||
|
||||
// writes the CDR to the disk
|
||||
bool ZipDir::CacheRW::WriteCDR(FILE* fTarget)
|
||||
{
|
||||
if (!fTarget)
|
||||
return false;
|
||||
|
||||
if (fseek(fTarget, m_lCDROffset, SEEK_SET))
|
||||
return false;
|
||||
|
||||
FileRecordList arrFiles(GetRoot());
|
||||
//arrFiles.SortByFileOffset();
|
||||
size_t nSizeCDR = arrFiles.GetStats().nSizeCDR;
|
||||
void* pCDR = m_pHeap->Alloc(nSizeCDR, "ZipDir::CacheRW::WriteCDR");
|
||||
size_t nSizeCDRSerialized = arrFiles.MakeZipCDR(m_lCDROffset,pCDR);
|
||||
assert (nSizeCDRSerialized == nSizeCDR);
|
||||
size_t nWriteRes = fwrite (pCDR, nSizeCDR, 1, fTarget);
|
||||
m_pHeap->Free(pCDR);
|
||||
return nWriteRes == 1;
|
||||
}
|
||||
|
||||
// generates random file name
|
||||
string ZipDir::CacheRW::GetRandomName(int nAttempt)
|
||||
{
|
||||
if (nAttempt)
|
||||
{
|
||||
char szBuf[8];
|
||||
int i;
|
||||
for (i = 0; i < sizeof(szBuf)-1;++i)
|
||||
{
|
||||
int r = rand()%(10 + 'z' - 'a' + 1);
|
||||
szBuf[i] = r > 9 ? (r-10)+'a' : '0' + r;
|
||||
}
|
||||
szBuf[i] = '\0';
|
||||
return szBuf;
|
||||
}
|
||||
else
|
||||
return string();
|
||||
}
|
||||
|
||||
bool ZipDir::CacheRW::RelinkZip()
|
||||
{
|
||||
for (int nAttempt = 0; nAttempt < 32; ++nAttempt)
|
||||
{
|
||||
string strNewFilePath = m_strFilePath + "$" + GetRandomName(nAttempt);
|
||||
if (GetFileAttributes (strNewFilePath.c_str()) != -1)
|
||||
continue; // we don't want to overwrite the old temp files for safety reasons
|
||||
|
||||
FILE* f = fopen (strNewFilePath.c_str(), "wb");
|
||||
if (f)
|
||||
{
|
||||
bool bOk = RelinkZip(f);
|
||||
fclose (f); // we don't need the temporary file handle anyway
|
||||
|
||||
if (!bOk)
|
||||
{
|
||||
// we don't need the temporary file
|
||||
unlink (strNewFilePath.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// we successfully relinked, now copy the temporary file to the original file
|
||||
fclose (m_pFile);
|
||||
m_pFile = NULL;
|
||||
|
||||
remove (m_strFilePath.c_str());
|
||||
if (rename (strNewFilePath.c_str(), m_strFilePath.c_str()) == 0)
|
||||
{
|
||||
// successfully renamed - reopen
|
||||
m_pFile = fopen (m_strFilePath.c_str(), "r+b");
|
||||
return m_pFile == NULL;
|
||||
}
|
||||
else
|
||||
{
|
||||
// could not rename
|
||||
|
||||
//m_pFile = fopen (strNewFilePath.c_str(), "r+b");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// couldn't open temp file
|
||||
return false;
|
||||
}
|
||||
|
||||
bool ZipDir::CacheRW::RelinkZip(FILE* fTmp)
|
||||
{
|
||||
FileRecordList arrFiles(GetRoot());
|
||||
arrFiles.SortByFileOffset();
|
||||
FileRecordList::ZipStats Stats = arrFiles.GetStats();
|
||||
|
||||
// we back up our file entries, because we'll need to restore them
|
||||
// in case the operation fails
|
||||
std::vector<FileEntry> arrFileEntryBackup;
|
||||
arrFiles.Backup (arrFileEntryBackup);
|
||||
|
||||
// this is the set of files that are to be written out - compressed data and the file record iterator
|
||||
std::vector<FileDataRecordPtr> queFiles;
|
||||
queFiles.reserve (g_nMaxItemsRelinkBuffer);
|
||||
|
||||
// the total size of data in the queue
|
||||
unsigned nQueueSize = 0;
|
||||
|
||||
for (FileRecordList::iterator it = arrFiles.begin(); it != arrFiles.end(); ++it)
|
||||
{
|
||||
// find the file data offset
|
||||
if (ZD_ERROR_SUCCESS != Refresh (it->pFileEntry))
|
||||
return false;
|
||||
|
||||
// go to the file data
|
||||
if (fseek (m_pFile, it->pFileEntry->nFileDataOffset, SEEK_SET) != 0)
|
||||
return false;
|
||||
|
||||
// allocate memory for the file compressed data
|
||||
FileDataRecordPtr pFile = FileDataRecord::New (*it, m_pHeap);
|
||||
|
||||
// read the compressed data
|
||||
if (it->pFileEntry->desc.lSizeCompressed && fread (pFile->GetData(), it->pFileEntry->desc.lSizeCompressed, 1, m_pFile) != 1)
|
||||
return false;
|
||||
|
||||
// put the file into the queue for copying (writing)
|
||||
queFiles.push_back(pFile);
|
||||
nQueueSize += it->pFileEntry->desc.lSizeCompressed;
|
||||
|
||||
// if the queue is big enough, write it out
|
||||
if(nQueueSize > g_nSizeRelinkBuffer || queFiles.size() >= g_nMaxItemsRelinkBuffer)
|
||||
{
|
||||
nQueueSize = 0;
|
||||
if (!WriteZipFiles(queFiles, fTmp))
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
if (!WriteZipFiles(queFiles, fTmp))
|
||||
return false;
|
||||
|
||||
ZipFile::ulong lOldCDROffset = m_lCDROffset;
|
||||
// the file data has now been written out. Now write the CDR
|
||||
m_lCDROffset = ftell(fTmp);
|
||||
if (m_lCDROffset >= 0 && WriteCDR(fTmp) && 0 == fflush (fTmp))
|
||||
// the new file positions are already there - just discard the backup and return
|
||||
return true;
|
||||
// recover from backup
|
||||
arrFiles.Restore (arrFileEntryBackup);
|
||||
m_lCDROffset = lOldCDROffset;
|
||||
return false;
|
||||
}
|
||||
|
||||
// writes out the file data in the queue into the given file. Empties the queue
|
||||
bool ZipDir::CacheRW::WriteZipFiles(std::vector<FileDataRecordPtr>& queFiles, FILE* fTmp)
|
||||
{
|
||||
for (std::vector<FileDataRecordPtr>::iterator it = queFiles.begin(); it != queFiles.end(); ++it)
|
||||
{
|
||||
// set the new header offset to the file entry - we won't need it
|
||||
(*it)->pFileEntry->nFileHeaderOffset = ftell (fTmp);
|
||||
|
||||
// while writing the local header, the data offset will also be calculated
|
||||
if (ZD_ERROR_SUCCESS != WriteLocalHeader(fTmp, (*it)->pFileEntry, (*it)->strPath.c_str()))
|
||||
return false;;
|
||||
|
||||
// write the compressed file data
|
||||
if ((*it)->pFileEntry->desc.lSizeCompressed && fwrite ((*it)->GetData(), (*it)->pFileEntry->desc.lSizeCompressed, 1, fTmp) != 1)
|
||||
return false;
|
||||
|
||||
assert ((*it)->pFileEntry->nEOFOffset == ftell (fTmp));
|
||||
}
|
||||
queFiles.clear();
|
||||
queFiles.reserve (g_nMaxItemsRelinkBuffer);
|
||||
return true;
|
||||
}
|
||||
Reference in New Issue
Block a user