Skip to content

Commit c8820ef

Browse files
aalhossaryaalhossary
committed
Test the density fetching, offline and against the live services
Forty-three unit tests run with no network at all. The chain tests use stub providers, so the behaviour that matters can be pinned exactly: a 404 falls through to the next source, a too-large map falls through as well, and anything else aborts. That last one is the point of the design - reporting a dropped connection as "this entry has no density" would be worse than failing. The offline set also pins the two-character directory rule against both spellings of an entry (1cbs and pdb_00001cbs must land in "cb", not "db"), that every source and kind combination maps to a distinct file, and that a LOCAL_ONLY request is served entirely from disk with every server pointed at a dead port. Six integration tests exercise the real services for a couple of megabytes in total. The cryo-EM path is covered without downloading the 116 MB map: the EMDB entry is resolved, the author contour level checked against its known value, and the size guard then declines the full map before any of its body is transferred. The coefficient test corrupts a downloaded file afterwards to confirm the ETag-derived MD5 actually catches it rather than merely being recorded. Writing the header check turned up a real bug: isCcp4 rejected any file shorter than a CCP4 header, but a gzipped map compresses to a small fraction of the header it contains, so small EMDB maps would have been rejected as invalid. The length shortcut is gone; reading decides it.
1 parent e20069e commit c8820ef

10 files changed

Lines changed: 1115 additions & 1 deletion

File tree

‎biojava-integrationtest/src/test/java/org/biojava/nbio/structure/test/io/density/DensityMapIntegrationTest.java‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
/**
2+
* BioJava development code
3+
*
4+
* This code may be freely distributed and modified under the terms of the GNU
5+
* Lesser General Public Licence. This should be distributed with the code. If
6+
* you do not have a copy, see:
7+
*
8+
* http://www.gnu.org/copyleft/lesser.html
9+
*
10+
* Copyright for this code is held jointly by the individual authors. These
11+
* should be listed in @author doc comments.
12+
*
13+
* For more information on the BioJava project and its aims, or to join the
14+
* biojava-l mailing list, visit the home page at:
15+
*
16+
* http://www.biojava.org/
17+
*/
18+
package org.biojava.nbio.structure.test.io.density;
19+
20+
import static org.junit.Assert.assertEquals;
21+
import static org.junit.Assert.assertFalse;
22+
import static org.junit.Assert.assertNotNull;
23+
import static org.junit.Assert.assertTrue;
24+
import static org.junit.Assert.fail;
25+
26+
import java.io.File;
27+
import java.io.IOException;
28+
import java.nio.file.Files;
29+
import java.util.Arrays;
30+
import java.util.List;
31+
32+
import org.biojava.nbio.core.util.FileDownloadUtils;
33+
import org.biojava.nbio.structure.PdbId;
34+
import org.biojava.nbio.structure.io.density.Ccp4Header;
35+
import org.biojava.nbio.structure.io.density.DensityFileFormat;
36+
import org.biojava.nbio.structure.io.density.DensityMapCache;
37+
import org.biojava.nbio.structure.io.density.DensityMapKind;
38+
import org.biojava.nbio.structure.io.density.DensityMapRequest;
39+
import org.biojava.nbio.structure.io.density.DensityMapResult;
40+
import org.biojava.nbio.structure.io.density.DensityMapSource;
41+
import org.biojava.nbio.structure.io.density.NoDensityMapException;
42+
import org.junit.After;
43+
import org.junit.Before;
44+
import org.junit.Test;
45+
46+
/**
47+
* Density fetching against the real services.
48+
* <p>
49+
* Deliberately frugal: the entries chosen keep a full run to a couple of
50+
* megabytes plus a few small metadata calls. In particular the cryo-EM path is
51+
* exercised with the size limit set so low that the 116 MB map is declined
52+
* before any of its body is transferred, which tests the whole resolution and
53+
* guard sequence without the download.
54+
*
55+
* @author Amr ALHOSSARY
56+
* @since 7.3.0
57+
*/
58+
public class DensityMapIntegrationTest {
59+
60+
private File cacheRoot;
61+
private DensityMapCache cache;
62+
63+
@Before
64+
public void setUp() throws IOException {
65+
cacheRoot = Files.createTempDirectory("bj-density-it").toFile();
66+
cache = new DensityMapCache(cacheRoot.getAbsolutePath());
67+
}
68+
69+
@After
70+
public void tearDown() throws IOException {
71+
FileDownloadUtils.deleteDirectory(cacheRoot.toPath());
72+
}
73+
74+
/** The default path for an X-ray entry: the smallest source answers first. */
75+
@Test
76+
public void fetchesAnXrayMapFromTheFirstSourceTried() throws IOException {
77+
DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
78+
79+
assertEquals(DensityMapSource.RCSB_VOLUME_SERVER, result.getSource());
80+
assertEquals(DensityMapKind.TWO_FO_FC, result.getKind());
81+
assertTrue(result.isRenderable());
82+
assertFalse(result.isFromCache());
83+
assertTrue(result.getFileSizeBytes() > 1024);
84+
assertTrue("a .meta sidecar makes the result reconstructible offline",
85+
DensityMapResult.metaFileFor(result.getFile()).isFile());
86+
87+
// second call must come from the cache without another download
88+
DensityMapResult again = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
89+
assertTrue(again.isFromCache());
90+
assertEquals(result.getFile(), again.getFile());
91+
}
92+
93+
/**
94+
* Both map kinds come out of one download, and the difference map is presented
95+
* under the companion name that makes Jmol read the other data block.
96+
*/
97+
@Test
98+
public void bothKindsShareASingleDownload() throws IOException {
99+
DensityMapResult twoFoFc = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
100+
DensityMapResult foFc = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.FO_FC);
101+
102+
assertEquals(DensityMapKind.FO_FC, foFc.getKind());
103+
assertFalse("the difference map needs its own file name", twoFoFc.getFile().equals(foFc.getFile()));
104+
assertTrue("the marker has to be in the name for Jmol to select the FO-FC block",
105+
foFc.getFile().getName().contains("&diff=1"));
106+
assertEquals("both names must address the same bytes",
107+
twoFoFc.getFileSizeBytes(), foFc.getFileSizeBytes());
108+
}
109+
110+
/** PDBe serves real CCP4 files, which the header check should recognise. */
111+
@Test
112+
public void pdbeServesAGenuineCcp4Map() throws IOException {
113+
cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(DensityMapSource.PDBE_CCP4));
114+
DensityMapResult result = cache.getDensityMap(new PdbId("1cbs"), DensityMapKind.TWO_FO_FC);
115+
116+
assertEquals(DensityMapSource.PDBE_CCP4, result.getSource());
117+
assertEquals(DensityFileFormat.CCP4, result.getFormat());
118+
assertTrue("the CCP4 stamp should be present at byte 208", Ccp4Header.isCcp4(result.getFile()));
119+
assertTrue(FileDownloadUtils.validateFile(result.getFile()));
120+
}
121+
122+
/**
123+
* The whole cryo-EM route: resolve the EMDB entry, pick up the author contour
124+
* level, and decline the full map on size without transferring it.
125+
*/
126+
@Test
127+
public void resolvesCryoEmEntriesAndHonoursTheSizeLimit() throws IOException {
128+
List<String> emdbIds = cache.getEmdbResolver().getEmdbIds(new PdbId("6hu9"));
129+
assertEquals(Arrays.asList("EMD-0262"), emdbIds);
130+
131+
DensityMapResult result = cache.getDensityMap(new PdbId("6hu9"), DensityMapKind.AUTO);
132+
assertEquals(DensityMapKind.EM, result.getKind());
133+
assertEquals("EMD-0262", result.getEmdbId());
134+
assertNotNull("EM maps need the author contour level to be displayed properly",
135+
result.getRecommendedContourLevel());
136+
assertEquals(0.0263, result.getRecommendedContourLevel(), 1e-6);
137+
assertNotNull(result.getContourInSigma());
138+
139+
// With only the full archive enabled and a tiny ceiling, the guard must fire
140+
// rather than pulling down 116 MB.
141+
DensityMapCache strict = new DensityMapCache(cacheRoot.getAbsolutePath());
142+
strict.setSourceChain(DensityMapKind.EM, Arrays.asList(DensityMapSource.EMDB_MAP));
143+
strict.setMaxDownloadBytes(1024);
144+
try {
145+
strict.getDensityMap(DensityMapRequest.builder(new PdbId("6hu9")).kind(DensityMapKind.EM).build());
146+
fail("the size guard should have declined the full EMDB map");
147+
} catch (NoDensityMapException e) {
148+
assertTrue(e.getAttempts().get(DensityMapSource.EMDB_MAP).contains("too large"));
149+
}
150+
}
151+
152+
/** 4HHB was deposited in 1984 without structure factors, so nothing has a map for it. */
153+
@Test
154+
public void reportsWhyAnEntryHasNoDensity() throws IOException {
155+
cache.setSourceEnabled(DensityMapSource.WWPDB_MAP_COEFFICIENTS, true);
156+
try {
157+
cache.getDensityMap(new PdbId("4hhb"), DensityMapKind.AUTO);
158+
fail("4hhb has no deposited structure factors");
159+
} catch (NoDensityMapException e) {
160+
assertFalse(e.getAttempts().isEmpty());
161+
assertTrue(e.getAttempts().values().stream().anyMatch(r -> r.contains("404")));
162+
}
163+
}
164+
165+
/**
166+
* The wwPDB servers return the content MD5 as the ETag, so a coefficient
167+
* download is checksum-verified without a separate hash file.
168+
*/
169+
@Test
170+
public void mapCoefficientsArriveWithAVerifiableChecksum() throws IOException {
171+
cache.setSourceEnabled(DensityMapSource.WWPDB_MAP_COEFFICIENTS, true);
172+
cache.setSourceChain(DensityMapKind.TWO_FO_FC, Arrays.asList(DensityMapSource.WWPDB_MAP_COEFFICIENTS));
173+
174+
DensityMapResult result = cache.getDensityMap(DensityMapRequest.builder(new PdbId("1cbs"))
175+
.kind(DensityMapKind.TWO_FO_FC)
176+
.allowNonRenderableFormats(true)
177+
.build());
178+
179+
assertEquals(DensityMapSource.WWPDB_MAP_COEFFICIENTS, result.getSource());
180+
assertFalse("structure factors are not a map and must not claim to be renderable",
181+
result.isRenderable());
182+
183+
File hashFile = new File(result.getFile().getParentFile(), result.getFile().getName() + ".hash_MD5");
184+
assertTrue("an MD5 should have been recorded from the ETag", hashFile.isFile());
185+
assertTrue(FileDownloadUtils.validateFile(result.getFile()));
186+
187+
// corrupt it and confirm the checksum actually catches it
188+
Files.write(result.getFile().toPath(), new byte[] {0, 1, 2, 3});
189+
assertFalse(FileDownloadUtils.validateFile(result.getFile()));
190+
}
191+
}

‎biojava-structure/src/main/java/org/biojava/nbio/structure/io/density/Ccp4Header.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,12 @@ private Ccp4Header() {
6868
* @throws IOException if the file could not be read
6969
*/
7070
public static boolean isCcp4(File file) throws IOException {
71-
if (file == null || !file.isFile() || file.length() < HEADER_BYTES) {
71+
if (file == null || !file.isFile()) {
7272
return false;
7373
}
74+
// Deliberately no shortcut on file.length(): a gzipped map compresses to far
75+
// less than the size of the header it contains, so a length test here would
76+
// reject perfectly good small maps. Reading decides it instead.
7477
try (InputStream in = openPossiblyGzipped(file)) {
7578
return isCcp4(in);
7679
}

‎biojava-structure/src/test/java/org/biojava/nbio/structure/io/density/TestCcp4Header.java‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
/**
2+
* BioJava development code
3+
*
4+
* This code may be freely distributed and modified under the terms of the GNU
5+
* Lesser General Public Licence. This should be distributed with the code. If
6+
* you do not have a copy, see:
7+
*
8+
* http://www.gnu.org/copyleft/lesser.html
9+
*
10+
* Copyright for this code is held jointly by the individual authors. These
11+
* should be listed in @author doc comments.
12+
*
13+
* For more information on the BioJava project and its aims, or to join the
14+
* biojava-l mailing list, visit the home page at:
15+
*
16+
* http://www.biojava.org/
17+
*/
18+
package org.biojava.nbio.structure.io.density;
19+
20+
import static org.junit.Assert.assertFalse;
21+
import static org.junit.Assert.assertTrue;
22+
23+
import java.io.ByteArrayOutputStream;
24+
import java.io.File;
25+
import java.io.IOException;
26+
import java.nio.charset.StandardCharsets;
27+
import java.nio.file.Files;
28+
import java.util.zip.GZIPOutputStream;
29+
30+
import org.biojava.nbio.core.util.FileDownloadUtils;
31+
import org.junit.After;
32+
import org.junit.Before;
33+
import org.junit.Test;
34+
35+
/**
36+
* The CCP4 header check that keeps a server's error page out of the cache.
37+
*
38+
* @author Amr ALHOSSARY
39+
* @since 7.3.0
40+
*/
41+
public class TestCcp4Header {
42+
43+
private File dir;
44+
45+
@Before
46+
public void setUp() throws IOException {
47+
dir = Files.createTempDirectory("bj-ccp4").toFile();
48+
}
49+
50+
@After
51+
public void tearDown() throws IOException {
52+
FileDownloadUtils.deleteDirectory(dir.toPath());
53+
}
54+
55+
/** A minimal file carrying the stamp at the offset a real CCP4 header uses. */
56+
private static byte[] fakeMap() {
57+
byte[] bytes = new byte[2048];
58+
byte[] stamp = Ccp4Header.MAP_STAMP.getBytes(StandardCharsets.US_ASCII);
59+
System.arraycopy(stamp, 0, bytes, Ccp4Header.MAP_STAMP_OFFSET, stamp.length);
60+
return bytes;
61+
}
62+
63+
private File write(String name, byte[] content) throws IOException {
64+
File f = new File(dir, name);
65+
Files.write(f.toPath(), content);
66+
return f;
67+
}
68+
69+
@Test
70+
public void recognisesAMapByItsStamp() throws IOException {
71+
assertTrue(Ccp4Header.isCcp4(write("good.ccp4", fakeMap())));
72+
}
73+
74+
@Test
75+
public void recognisesAGzippedMap() throws IOException {
76+
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
77+
try (GZIPOutputStream gz = new GZIPOutputStream(buffer)) {
78+
gz.write(fakeMap());
79+
}
80+
assertTrue("EMDB serves its maps gzipped", Ccp4Header.isCcp4(write("good.map.gz", buffer.toByteArray())));
81+
}
82+
83+
/**
84+
* The case this check exists for: a server answering with an error page and an
85+
* HTTP 200, which nothing else would catch.
86+
*/
87+
@Test
88+
public void rejectsAnHtmlErrorPage() throws IOException {
89+
StringBuilder html = new StringBuilder("<html><head><title>404 Not Found</title></head><body>");
90+
while (html.length() < 1500) {
91+
html.append("<p>The requested resource was not found on this server.</p>");
92+
}
93+
html.append("</body></html>");
94+
assertFalse(Ccp4Header.isCcp4(write("error.ccp4", html.toString().getBytes(StandardCharsets.UTF_8))));
95+
}
96+
97+
@Test
98+
public void rejectsRandomBytesAndShortFiles() throws IOException {
99+
byte[] noise = new byte[2048];
100+
for (int i = 0; i < noise.length; i++) {
101+
noise[i] = (byte) (i * 31);
102+
}
103+
assertFalse(Ccp4Header.isCcp4(write("noise.ccp4", noise)));
104+
assertFalse("a file shorter than the header cannot be a map",
105+
Ccp4Header.isCcp4(write("tiny.ccp4", new byte[10])));
106+
}
107+
108+
@Test
109+
public void quietVariantSwallowsUnreadableFiles() {
110+
assertFalse(Ccp4Header.isCcp4Quietly(new File(dir, "does-not-exist.ccp4")));
111+
}
112+
}

0 commit comments

Comments
 (0)