Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Abstract ref_to_m5 from cram_io #589

Open
wants to merge 2 commits into
base: develop
Choose a base branch
from
Open

Abstract ref_to_m5 from cram_io #589

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
59dc1f9
Abstract ref_to_m5 from cram_io
ThomasHickman Jul 13, 2018
55e9668
Add error messages for failed reference lookups
ThomasHickman Jul 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ lib*.so.*
/test/test-regidx
/test/test-vcf-api
/test/test-vcf-sweep
/test/test-ref
/test/test_view
/test/thrash_threads[1-6]
/test/*.tmp
Expand Down
40 changes: 32 additions & 8 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ BUILT_TEST_PROGRAMS = \
test/hfile \
test/sam \
test/test_bgzf \
test/test-ref \
test/test_realn \
test/test-regidx \
test/test_view \
Expand Down Expand Up @@ -140,6 +141,7 @@ LIBHTS_OBJS = \
multipart.o \
probaln.o \
realn.o \
ref.o \
regidx.o \
sam.o \
synced_bcf_reader.o \
Expand All @@ -158,8 +160,6 @@ LIBHTS_OBJS = \
cram/cram_samtools.o \
cram/cram_stats.o \
cram/files.o \
cram/mFILE.o \
cram/open_trace_file.o \
cram/pooled_alloc.o \
cram/rANS_static.o \
cram/sam_header.o \
Expand All @@ -174,11 +174,12 @@ cram_misc_h = cram/misc.h $(cram_os_h)
cram_os_h = cram/os.h $(htslib_hts_endian_h)
cram_sam_header_h = cram/sam_header.h cram/string_alloc.h cram/pooled_alloc.h $(htslib_khash_h) $(htslib_kstring_h)
cram_samtools_h = cram/cram_samtools.h $(htslib_sam_h) $(cram_sam_header_h)
cram_structs_h = cram/cram_structs.h $(htslib_thread_pool_h) cram/string_alloc.h cram/mFILE.h $(htslib_khash_h)
cram_open_trace_file_h = cram/open_trace_file.h cram/mFILE.h
bcf_sr_sort_h = bcf_sr_sort.h $(htslib_synced_bcf_reader_h) $(htslib_kbitset_h)
cram_structs_h = cram/cram_structs.h $(htslib_thread_pool_h) cram/string_alloc.h $(htslib_khash_h)

hfile_internal_h = hfile_internal.h $(htslib_hfile_h) $(textutils_internal_h)
hts_internal_h = hts_internal.h $(htslib_hts_h) $(textutils_internal_h)
ref_internal_h = ref_internal.h $(htslib_hfile_h) cram/mFILE.h
textutils_internal_h = textutils_internal.h $(htslib_kstring_h)
thread_pool_internal_h = thread_pool_internal.h $(htslib_thread_pool_h)

Expand Down Expand Up @@ -316,18 +317,17 @@ plugin.o plugin.pico: plugin.c config.h $(hts_internal_h) $(htslib_kstring_h)
probaln.o probaln.pico: probaln.c config.h $(htslib_hts_h)
realn.o realn.pico: realn.c config.h $(htslib_hts_h) $(htslib_sam_h)
textutils.o textutils.pico: textutils.c config.h $(htslib_hfile_h) $(htslib_kstring_h) $(hts_internal_h)
ref.o ref.pico: ref.c config.h $(htslib_kstring_h) $(htslib_bgzf_h) $(cram_h) $(cram_io_h) $(htslib_hfile_h)

cram/cram_codecs.o cram/cram_codecs.pico: cram/cram_codecs.c config.h $(cram_h)
cram/cram_decode.o cram/cram_decode.pico: cram/cram_decode.c config.h $(cram_h) $(cram_os_h) $(htslib_hts_h)
cram/cram_encode.o cram/cram_encode.pico: cram/cram_encode.c config.h $(cram_h) $(cram_os_h) $(htslib_hts_h) $(htslib_hts_endian_h)
cram/cram_external.o cram/cram_external.pico: cram/cram_external.c config.h $(htslib_hfile_h) $(cram_h)
cram/cram_index.o cram/cram_index.pico: cram/cram_index.c config.h $(htslib_bgzf_h) $(htslib_hfile_h) $(hts_internal_h) $(cram_h) $(cram_os_h)
cram/cram_io.o cram/cram_io.pico: cram/cram_io.c config.h os/lzma_stub.h $(cram_h) $(cram_os_h) $(htslib_hts_h) $(cram_open_trace_file_h) cram/rANS_static.h $(htslib_hfile_h) $(htslib_bgzf_h) $(htslib_faidx_h) $(hts_internal_h)
cram/cram_io.o cram/cram_io.pico: cram/cram_io.c config.h os/lzma_stub.h $(cram_h) $(cram_os_h) $(htslib_hts_h) $(htslib_ref_h) cram/rANS_static.h $(htslib_hfile_h) $(htslib_bgzf_h) $(htslib_faidx_h) $(hts_internal_h)
cram/cram_samtools.o cram/cram_samtools.pico: cram/cram_samtools.c config.h $(cram_h) $(htslib_sam_h)
cram/cram_stats.o cram/cram_stats.pico: cram/cram_stats.c config.h $(cram_h) $(cram_os_h)
cram/files.o cram/files.pico: cram/files.c config.h $(cram_misc_h)
cram/mFILE.o cram/mFILE.pico: cram/mFILE.c config.h $(htslib_hts_log_h) $(cram_os_h) cram/mFILE.h
cram/open_trace_file.o cram/open_trace_file.pico: cram/open_trace_file.c config.h $(cram_os_h) $(cram_open_trace_file_h) $(cram_misc_h) $(htslib_hfile_h) $(htslib_hts_log_h)
cram/pooled_alloc.o cram/pooled_alloc.pico: cram/pooled_alloc.c config.h cram/pooled_alloc.h $(cram_misc_h)
cram/rANS_static.o cram/rANS_static.pico: cram/rANS_static.c config.h cram/rANS_static.h cram/rANS_byte.h
cram/sam_header.o cram/sam_header.pico: cram/sam_header.c config.h $(htslib_hts_log_h) $(cram_sam_header_h) cram/string_alloc.h
Expand All @@ -354,7 +354,7 @@ tabix.o: tabix.c config.h $(htslib_tbx_h) $(htslib_sam_h) $(htslib_vcf_h) $(htsl
#
# If using MSYS, avoid poor shell expansion via:
# MSYS2_ARG_CONV_EXCL="*" make check
check test: $(BUILT_PROGRAMS) $(BUILT_TEST_PROGRAMS)
check test: $(BUILT_PROGRAMS) $(BUILT_TEST_PROGRAMS) test_ebi_fetch
test/hts_endian
test/fieldarith test/fieldarith.sam
test/hfile
Expand All @@ -364,6 +364,26 @@ check test: $(BUILT_PROGRAMS) $(BUILT_TEST_PROGRAMS)
test/test-regidx
cd test && REF_PATH=: ./test.pl $${TEST_OPTS:-}

test_ebi_fetch:
@if [ "$(ENABLE_HTTP_TESTS)" = true ] ; then \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please add this script to a separate .sh file and run it from here (similarly to what has been done for the tabix test above).

export REF_CACHE=`mktemp -d` && \
echo "test ref: fetching" && \
test/test-ref && \
echo "test ref: fetch from cache" && \
test/test-ref && \
rm -rf $$REF_CACHE; \
export REF_CACHE=`mktemp -d` && \
test/test_view -t test/xx.fa -S -C test/xx#rg.sam > test/xx#rg.tmp.cram && \
echo "test view: fetching" && \
test/test_view -D test/xx#rg.tmp.cram > /dev/null && \
echo "test view: fetch from cache" && \
test/test_view -D test/xx#rg.tmp.cram > /dev/null && \
rm -rf $$REF_CACHE; \
else \
echo "Warning: testing of EBI fetching disabled"; \
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Complete the message with instructions about how to enable this test.

fi


test/hts_endian: test/hts_endian.o
$(CC) $(LDFLAGS) -o $@ test/hts_endian.o $(LIBS)

Expand All @@ -379,6 +399,9 @@ test/sam: test/sam.o libhts.a
test/test_bgzf: test/test_bgzf.o libhts.a
$(CC) $(LDFLAGS) -o $@ test/test_bgzf.o libhts.a -lz $(LIBS) -lpthread

test/test-ref: test/test-ref.o libhts.a
$(CC) $(LDFLAGS) -o $@ test/test-ref.o libhts.a $(LIBS) -lpthread

test/test_realn: test/test_realn.o libhts.a
$(CC) $(LDFLAGS) -o $@ test/test_realn.o libhts.a $(LIBS) -lpthread

Expand Down Expand Up @@ -406,6 +429,7 @@ test/hfile.o: test/hfile.c config.h $(htslib_hfile_h) $(htslib_hts_defs_h)
test/sam.o: test/sam.c config.h $(htslib_hts_defs_h) $(htslib_sam_h) $(htslib_faidx_h) $(htslib_kstring_h)
test/test_bgzf.o: test/test_bgzf.c config.h $(htslib_bgzf_h) $(htslib_hfile_h) $(hfile_internal_h)
test/test-realn.o: test/test_realn.c config.h $(htslib_hts_h) $(htslib_sam_h) $(htslib_faidx_h)
test/test-ref.o: test/test-ref.c $(htslib_bgzf_h)
test/test-regidx.o: test/test-regidx.c config.h $(htslib_regidx_h) $(hts_internal_h)
test/test_view.o: test/test_view.c config.h $(cram_h) $(htslib_sam_h)
test/test-vcf-api.o: test/test-vcf-api.c config.h $(htslib_hts_h) $(htslib_vcf_h) $(htslib_kstring_h) $(htslib_kseq_h)
Expand Down
1 change: 1 addition & 0 deletions config.mk.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ LIBS = @LIBS@

PLATFORM = @PLATFORM@
PLUGIN_EXT = @PLUGIN_EXT@
ENABLE_HTTP_TESTS = @ENABLE_HTTP_TESTS@

# Lowercase here indicates these are "local" to config.mk
plugin_OBJS =
Expand Down
12 changes: 11 additions & 1 deletion configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,11 @@ AC_ARG_ENABLE([s3],
[support Amazon AWS S3 URLs])],
[], [enable_s3=check])

AC_ARG_ENABLE([http-tests],
[AS_HELP_STRING([--disable-http-tests],
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Describe the option --disable-http-tests in the Configuration section of the INSTALL file.

[disables tests that require querying to a http server])],
[disable_http_tests=yes], [disable_http_tests=no])
Copy link
Contributor

@valeriuo valeriuo Dec 20, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also, it was decided that HTTP tests should be disabled by default. So the option should be renamed --enable-http-tests and the associated variable should only be set to true when this option is selected.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. Although @daviesrob may have a trivial perl http server (with no complex perl package dependencies) that could permit testing against a local server without needing network access. (I'm a bit wary of going that route though unless it's very little extra code to maintain.)


test -n "$host_alias" || host_alias=unknown-`uname -s`
AC_MSG_CHECKING([shared library type for $host_alias])
case $host_alias in
Expand Down Expand Up @@ -293,10 +298,14 @@ Either configure with --without-libdeflate or resolve this error to build
HTSlib.])])])])

libcurl=disabled
ENABLE_HTTP_TESTS=false
if test "$enable_libcurl" != no; then
AC_CHECK_LIB([curl], [curl_easy_pause],
[AC_DEFINE([HAVE_LIBCURL], 1, [Define if libcurl file access is enabled.])
libcurl=enabled],
libcurl=enabled
if test "$disable_http_tests" = no; then
ENABLE_HTTP_TESTS=true
fi ],
[AC_CHECK_LIB([curl], [curl_easy_init],
[message="library is too old (7.18+ required)"],
[message="library not found"])
Expand All @@ -322,6 +331,7 @@ dnl -lcurl is only needed for static linking if hfile_libcurl is not a plugin
fi
fi
AC_SUBST([libcurl])
AC_SUBST([ENABLE_HTTP_TESTS])

gcs=disabled
if test "$enable_gcs" != no; then
Expand Down
3 changes: 0 additions & 3 deletions cram/cram_index.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -588,9 +588,6 @@ int cram_index_build(cram_fd *fd, const char *fn_base, const char *fn_idx) {
BGZF *fp;
kstring_t fn_idx_str = {0};

// Useful for cram_index_build_multiref
cram_set_option(fd, CRAM_OPT_REQUIRED_FIELDS, SAM_RNAME | SAM_POS | SAM_CIGAR);

if (! fn_idx) {
kputs(fn_base, &fn_idx_str);
kputs(".crai", &fn_idx_str);
Expand Down
Loading