WIP Start adding ReadsHtsgetDataSource tests

broadinstitute · andersleung · Jun 16, 2020 · Jun 16, 2020 · Jun 16, 2020 · Jun 17, 2020
commit 890da677da65d5d6b008f55c4565cf8ba10afc21
diff --git a/.travis.yml b/.travis.yml
@@ -4,6 +4,8 @@ group: travis_lts
 git:
   depth: 9999999
   lfs_skip_smudge: true
+services:
+  - docker
 jdk:
   - openjdk8
 env:
@@ -101,6 +103,8 @@ before_install:
 # http://docs.travis-ci.com/user/database-setup/#MySQL
 - sudo /etc/init.d/mysql stop
 - sudo /etc/init.d/postgresql stop
+- sudo bash scripts/htsgetScripts/launchDocker.sh
+
 install:
 - if [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
     echo "Can't run cloud tests without keys so don't bother building";

diff --git a/scripts/htsgetScripts/launchDocker.sh b/scripts/htsgetScripts/launchDocker.sh
@@ -0,0 +1,11 @@
+WORKING_DIR=/home/travis/build/broadinstitute
+
+echo "pwd is `pwd`"
+ls
+
+docker pull ga4gh/htsget-ref:1.1.0
+docker container run -d --name htsget-server -p 3000:3000 --env HTSGET_PORT=3000 --env HTSGET_HOST=http://localhost:3000 -v $WORKING_DIR/gatk/src/test/resources/org/broadinstitute/hellbender/engine:/data ga4gh/htsget-ref:1.1.0 ./htsref -config /data/htsget_config.json
+
+echo "Running docker containers"
+
+docker container ls -a
diff --git a/src/main/java/org/broadinstitute/hellbender/engine/ReadsHtsgetDataSource.java b/src/main/java/org/broadinstitute/hellbender/engine/ReadsHtsgetDataSource.java
@@ -29,9 +29,9 @@
  * Manages traversals and queries over sources of reads which are accessible via {@link GATKPath}s pointing to a file
  * behind an htsget server
  * (for now, BAM/CRAM files only).
- *
+ * <p>
  * Two basic operations are available:
- *
+ * <p>
  * -Iteration over all reads, optionally restricted to reads that overlap a set of intervals
  * -Targeted queries by one interval at a time
  */
@@ -48,18 +48,18 @@ public final class ReadsHtsgetDataSource implements ReadsDataSource {
     /**
      * Only reads that overlap these intervals (and unmapped reads, if {@link #traverseUnmapped} is set) will be returned
      * during a full iteration. Null if iteration is unbounded.
-     *
+     * <p>
      * Individual queries are unaffected by these intervals -- only traversals initiated via {@link #iterator} are affected.
      */
     private List<SimpleInterval> intervals;
 
     /**
      * If true, restrict traversals to unmapped reads (and reads overlapping any {@link #intervals}, if set).
      * False if iteration is unbounded or bounded only by our {@link #intervals}.
-     *
+     * <p>
      * Note that this setting covers only unmapped reads that have no position -- unmapped reads that are assigned the
      * position of their mates will be returned by queries overlapping that position.
-     *
+     * <p>
      * Individual queries are unaffected by this setting  -- only traversals initiated via {@link #iterator} are affected.
      */
     private boolean traverseUnmapped;
@@ -110,7 +110,7 @@ public ReadsHtsgetDataSource(final List<GATKPath> sources) {
     /**
      * Initialize this data source with a single SAM/BAM file and a custom SamReaderFactory
      *
-     * @param source path to SAM/BAM file, not null.
+     * @param source                 path to SAM/BAM file, not null.
      * @param customSamReaderFactory SamReaderFactory to use, if null a default factory with no reference and validation
      *                               stringency SILENT is used.
      */
@@ -121,7 +121,7 @@ public ReadsHtsgetDataSource(final GATKPath source, final SamReaderFactory custo
     /**
      * Initialize this data source with multiple SAM/BAM files and a custom SamReaderFactory
      *
-     * @param sources path to SAM/BAM file, not null.
+     * @param sources                path to SAM/BAM file, not null.
      * @param customSamReaderFactory SamReaderFactory to use, if null a default factory with no reference and validation
      *                               stringency SILENT is used.
      */
@@ -197,23 +197,23 @@ public ReadsHtsgetDataSource(final List<GATKPath> sources, final SamReaderFactor
     /**
      * Restricts a traversal of this data source via {@link #iterator} to only return reads that overlap the given intervals,
      * and to unmapped reads if specified.
-     *
+     * <p>
      * Calls to {@link #query} are not affected by this method.
      *
-     * @param intervals Our next full traversal will return reads overlapping these intervals
+     * @param intervals        Our next full traversal will return reads overlapping these intervals
      * @param traverseUnmapped Our next full traversal will return unmapped reads (this affects only unmapped reads that
      *                         have no position -- unmapped reads that have the position of their mapped mates will be
      *                         included if the interval overlapping that position is included).
      */
     @Override
     public void setTraversalBounds(final List<SimpleInterval> intervals, final boolean traverseUnmapped) {
-        this.intervals = intervals != null && ! intervals.isEmpty() ? intervals : null;
+        this.intervals = intervals != null && !intervals.isEmpty() ? intervals : null;
         this.traverseUnmapped = traverseUnmapped;
     }
 
     /**
      * @return True if traversals initiated via {@link #iterator} will be restricted to reads that overlap intervals
-     *         as configured via {@link #setTraversalBounds}, otherwise false
+     * as configured via {@link #setTraversalBounds}, otherwise false
      */
     @Override
     public boolean traversalIsBounded() {
@@ -222,6 +222,7 @@ public boolean traversalIsBounded() {
 
     /**
      * This data source can be queried even without index files
+     *
      * @return always true
      */
     @Override
@@ -234,7 +235,7 @@ public boolean isQueryableByInterval() {
      * iteration is limited to reads that overlap that set of intervals.
      *
      * @return An iterator over the reads in this data source, limited to reads that overlap the intervals supplied
-     *         via {@link #setTraversalBounds} (if intervals were provided)
+     * via {@link #setTraversalBounds} (if intervals were provided)
      */
     @Override
     @Nonnull
@@ -256,7 +257,7 @@ public Iterator<GATKRead> query(final SimpleInterval interval) {
 
     /**
      * @return An iterator over just the unmapped reads with no assigned position. This operation is not affected
-     *         by prior calls to {@link #setTraversalBounds}.
+     * by prior calls to {@link #setTraversalBounds}.
      */
     @Override
     public Iterator<GATKRead> queryUnmapped() {
@@ -388,7 +389,8 @@ public boolean filterOut(final SAMRecord first, final SAMRecord second) {
 
     /**
      * Wrap an iterator to allow us to free the backing SamReader without holding onto an explicit reference to it
-     * @param iterator the iterator to wrap
+     *
+     * @param iterator  the iterator to wrap
      * @param samReader the SamReader to close once the iterator has been used up
      * @return a wrapped CloseableIterator
      */

diff --git a/src/main/java/org/broadinstitute/hellbender/tools/htsgetreader/HtsgetRequest.java b/src/main/java/org/broadinstitute/hellbender/tools/htsgetreader/HtsgetRequest.java
@@ -39,8 +39,8 @@ public class HtsgetRequest {
 
     private static final ObjectMapper mapper = getObjectMapper();
 
-    final private URI endpoint;
-    final private String id;
+    private final URI endpoint;
+    private final String id;
 
     // Query parameters
     private HtsgetFormat format;
@@ -59,16 +59,11 @@ public HtsgetRequest(final URI endpoint, final String id) {
     }
 
     public HtsgetRequest(final GATKPath source) {
-        try {
-            final URI sourceURI = source.getURI();
-            this.endpoint = new URI("//" + sourceURI.getHost());
-            this.id = sourceURI.getPath();
-            this.fields = EnumSet.noneOf(HtsgetRequestField.class);
-            this.tags = new HashSet<>();
-            this.notags = new HashSet<>();
-        } catch (final URISyntaxException e) {
-            throw new UserException(source.toString(), e);
-        }
+        this.endpoint = source.getURI();
+        this.id = "";
+        this.fields = EnumSet.noneOf(HtsgetRequestField.class);
+        this.tags = new HashSet<>();
+        this.notags = new HashSet<>();
     }
 
     public URI getEndpoint() {
@@ -221,6 +216,7 @@ public URI toURI() {
         this.validateRequest();
         final UriBuilder builder = UriBuilder.fromUri(this.endpoint)
             .scheme("http")
+            .port(this.endpoint.getPort())
             .path(this.id);
 
         if (this.format != null) {