Created
February 14, 2014 01:31
-
-
Save leogrim/8994231 to your computer and use it in GitHub Desktop.
IO Helpers - Automated Backup and Restoration of Lucene Indices with Amazon S3 - http://eng.kifi.com/automated-backup-and-restoration-of-lucene-indices-with-amazon-s3/
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import java.io._ | |
import java.util.zip.{GZIPInputStream, GZIPOutputStream} | |
import org.apache.commons.compress.archivers.tar.{TarArchiveInputStream, TarArchiveEntry, TarArchiveOutputStream} | |
import org.apache.commons.io.{IOUtils, FileUtils} | |
object IO { | |
def addToArchive(tarArchive: TarArchiveOutputStream, file: File, base: String = ""): Unit = { | |
val entryName = base + file.getName | |
val entry = new TarArchiveEntry(file, entryName) | |
tarArchive.putArchiveEntry(entry) | |
entry.setSize(file.length()) | |
if (file.isFile) { FileUtils.copyFile(file, tarArchive) } | |
tarArchive.closeArchiveEntry() | |
if (file.isDirectory) file.listFiles().foreach(addToArchive(tarArchive, _, entryName + "/")) | |
} | |
def extractArchive(tarArchive: TarArchiveInputStream, destDir: String): Seq[File] = { | |
val files = new ListBuffer[File] | |
var entryOption = Option(tarArchive.getNextTarEntry) | |
while (entryOption.isDefined) { | |
entryOption.foreach { entry => | |
val file = new File(destDir, entry.getName) | |
files.append(file) | |
if (entry.isFile) { | |
val out = FileUtils.openOutputStream(file) | |
try { IOUtils.copyLarge(tarArchive, out, 0, entry.getSize) } | |
catch { case e: Throwable => files.foreach(_.delete()); throw e } // directories created by FileUtils.openOutputStream may not be cleaned up | |
finally { out.close() } | |
} | |
else { | |
try { FileUtils.forceMkdir(file) } | |
catch { case e: Throwable => files.foreach(_.delete()); throw e } // directories created by FileUtils.forceMkdir may not be cleaned up | |
} | |
} | |
entryOption = Option(tarArchive.getNextTarEntry) | |
} | |
files.toList | |
} | |
def compress(file: File, outputStream: OutputStream): Unit = { | |
val gZip = new GZIPOutputStream(outputStream) | |
val compressedTarArchive = new TarArchiveOutputStream(gZip) | |
compressedTarArchive.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_STAR) | |
addToArchive(compressedTarArchive, file) | |
compressedTarArchive.finish() | |
gZip.finish() | |
} | |
def compress(file: File, destDir: String = FileUtils.getTempDirectoryPath): File = { | |
val tarGz = new File(destDir, file.getName + ".tar.gz") | |
val out = FileUtils.openOutputStream(tarGz) | |
try { compress(file, out) } | |
catch { case e: Throwable => tarGz.delete(); throw e } | |
finally { out.close() } | |
tarGz | |
} | |
def uncompress(tarGzStream: InputStream, destDir: String): Unit = { | |
val uncompressedTarArchive = new TarArchiveInputStream(new GZIPInputStream(tarGzStream)) | |
extractArchive(uncompressedTarArchive, destDir) | |
} | |
def uncompress(tarGz: File, destDir: String): Unit = { | |
val inputStream = FileUtils.openInputStream(tarGz) | |
try { uncompress(inputStream, destDir) } | |
finally { inputStream.close() } | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment