Commit 1ed9ad72 authored by Rolf Krahl's avatar Rolf Krahl

Merge branch 'restorer_error'

parents bbaa368d db841cf0
......@@ -4,8 +4,10 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
......@@ -92,19 +94,34 @@ public class DsRestorer implements Runnable {
+ " files of total size " + size);
ZipInputStream zis = new ZipInputStream(Files.newInputStream(datasetCachePath));
ZipEntry ze = zis.getNextEntry();
Set<String> seen = new HashSet<>();
while (ze != null) {
String dfName = zipMapper.getFileName(ze.getName());
if (seen.contains(dfName)) {
throw new RuntimeException("Corrupt archive for " + dsInfo + ": duplicate entry " + dfName);
}
String location = nameToLocalMap.get(dfName);
if (location == null) {
throw new RuntimeException("Corrupt archive for " + dsInfo + ": spurious entry " + dfName);
}
mainStorageInterface.put(zis, location);
ze = zis.getNextEntry();
seen.add(dfName);
}
zis.close();
if (!seen.equals(nameToLocalMap.keySet())) {
throw new RuntimeException("Corrupt archive for " + dsInfo + ": missing entries");
}
Files.delete(datasetCachePath);
fsm.recordSuccess(dsInfo.getDsId());
logger.debug("Restore of " + dsInfo + " completed");
} catch (Exception e) {
fsm.recordFailure(dsInfo.getDsId());
logger.error("Restore of " + dsInfo + " failed due to " + e.getClass() + " " + e.getMessage());
try {
mainStorageInterface.delete(dsInfo);
} catch (IOException e2) {
}
} finally {
fsm.removeFromChanging(dsInfo);
lock.release();
......
......@@ -17,6 +17,8 @@
<li>Add new configuration properties delayDatasetWritesSeconds and
delayDatafileOperationsSeconds, replacing writeDelaySeconds. Deprecate
writeDelaySeconds. (Issue #94)</li>
<li>Error handling in DsRestorer in the case of a corrupt ZIP archive
(Issue #96).</li>
<li>Require ids.plugin 1.5.0.</li>
</ul>
......
package org.icatproject.ids.integration.two;
/*
* Test various error conditions in the DsRestorer caused by ZIP files
* in archive storage having unexpected content.
*/
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import java.util.zip.ZipOutputStream;
import org.junit.BeforeClass;
import org.junit.Test;
import org.icatproject.Datafile;
import org.icatproject.Dataset;
import org.icatproject.Facility;
import org.icatproject.IcatException_Exception;
import org.icatproject.Investigation;
import org.icatproject.ids.integration.BaseTest;
import org.icatproject.ids.integration.util.Setup;
import org.icatproject.ids.integration.util.client.DataSelection;
import org.icatproject.ids.integration.util.client.InternalException;
import org.icatproject.ids.integration.util.client.TestingClient.Status;
public class RestoreErrorsTest extends BaseTest {
@BeforeClass
public static void setup() throws Exception {
setup = new Setup("two.properties");
icatsetup();
}
/*
* Note that we cannot test for DUPLICATE_ENTRY here, because ZipOutputStream() won't allow
* us to create such a defective ZIP file. But that doesn't mean that this error cannot
* occur.
*/
private enum Defect {
NONE, MISSING_ENTRY, SPURIOUS_ENTRY, DUPLICATE_ENTRY
}
private void cloneZip(Path archivepath, Defect defect) throws IOException {
Path savepath = archivepath.getParent().resolve(".sav");
Files.move(archivepath, savepath);
try (ZipOutputStream zipout = new ZipOutputStream(Files.newOutputStream(archivepath))) {
try (ZipInputStream zipin = new ZipInputStream(Files.newInputStream(savepath))) {
ZipEntry entry = zipin.getNextEntry();
boolean first = true;
String entryName = "";
while (entry != null) {
if (first && defect == Defect.MISSING_ENTRY) {
entry = zipin.getNextEntry();
}
first = false;
entryName = entry.getName();
zipout.putNextEntry(new ZipEntry(entryName));
byte[] bytes = new byte[8192];
int length;
while ((length = zipin.read(bytes)) >= 0) {
zipout.write(bytes, 0, length);
}
zipout.closeEntry();
entry = zipin.getNextEntry();
}
}
if (defect == Defect.SPURIOUS_ENTRY) {
zipout.putNextEntry(new ZipEntry("ids/spurious_entry"));
byte[] bytes = new byte[64];
zipout.write(bytes, 0, 64);
zipout.closeEntry();
}
}
}
/*
* As a reference: a restore with no errors.
*/
@Test
public void restoreOk() throws Exception {
Long dsId = datasetIds.get(1);
Path archivefile = getFileOnArchiveStorage(dsId);
Path dirOnFastStorage = getDirOnFastStorage(dsId);
DataSelection selection = new DataSelection().addDataset(dsId);
cloneZip(archivefile, Defect.NONE);
testingClient.restore(sessionId, selection, 204);
waitForIds();
checkPresent(dirOnFastStorage);
}
/*
* A missing entry in the archive.
*/
@Test
public void restoreMissing() throws Exception {
Long dsId = datasetIds.get(1);
Path archivefile = getFileOnArchiveStorage(dsId);
Path dirOnFastStorage = getDirOnFastStorage(dsId);
DataSelection selection = new DataSelection().addDataset(dsId);
cloneZip(archivefile, Defect.MISSING_ENTRY);
testingClient.restore(sessionId, selection, 204);
waitForIds();
checkAbsent(dirOnFastStorage);
try {
testingClient.getStatus(sessionId, selection, null);
fail("Expected InternalException to be thrown.");
} catch (InternalException e) {
assertEquals("Restore failed", e.getMessage());
}
testingClient.reset(sessionId, selection, 204);
Status status = testingClient.getStatus(sessionId, selection, 200);
assertEquals(Status.ARCHIVED, status);
}
/*
* A spurious entry in the archive.
*/
@Test
public void restoreSpurious() throws Exception {
Long dsId = datasetIds.get(1);
Path archivefile = getFileOnArchiveStorage(dsId);
Path dirOnFastStorage = getDirOnFastStorage(dsId);
DataSelection selection = new DataSelection().addDataset(dsId);
cloneZip(archivefile, Defect.SPURIOUS_ENTRY);
testingClient.restore(sessionId, selection, 204);
waitForIds();
checkAbsent(dirOnFastStorage);
try {
testingClient.getStatus(sessionId, selection, null);
fail("Expected InternalException to be thrown.");
} catch (InternalException e) {
assertEquals("Restore failed", e.getMessage());
}
testingClient.reset(sessionId, selection, 204);
Status status = testingClient.getStatus(sessionId, selection, 200);
assertEquals(Status.ARCHIVED, status);
}
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment