PARQUET-679: Local Windows build and Appveyor support

Author: Max Risuhin <[email protected]>

Closes #313 from MaxRis/PARQUET-679 and squashes the following commits:

c5c4a8c [Max Risuhin] PARQUET-679: Local Windows build and Appveyor support

CMakeLists.txt

@@ -111,13 +111,15 @@ if ("${CMAKE_SOURCE_DIR}" STREQUAL "${CMAKE_CURRENT_SOURCE_DIR}")
     "Build our own zlib (some libz.a aren't configured for static linking)"
     ON)
   option(PARQUET_RPATH_ORIGIN
-    "Build Parquet libraries with RATH set to \$ORIGIN"
+    "Build Parquet libraries with RPATH set to \$ORIGIN"
     OFF)
   option(PARQUET_MINIMAL_DEPENDENCY
     "Depend only on Thirdparty headers to build libparquet. Always OFF if building binaries"
     OFF)
 endif()
 
+include(BuildUtils)
+
 if (PARQUET_BUILD_TESTS OR PARQUET_BUILD_EXECUTABLES OR PARQUET_BUILD_BENCHMARKS)
   set(PARQUET_BUILD_STATIC ON)
   set(PARQUET_MINIMAL_DEPENDENCY OFF)
@@ -283,6 +285,8 @@ function(ADD_PARQUET_TEST REL_TEST_NAME)
       COMPILE_FLAGS " -DPARQUET_VALGRIND")
     add_test(${TEST_NAME}
       valgrind --tool=memcheck --leak-check=full --error-exitcode=1 ${TEST_PATH})
+  elseif(MSVC)
+    add_test(${TEST_NAME} ${TEST_PATH})
   else()
     add_test(${TEST_NAME}
         ${BUILD_SUPPORT_DIR}/run-test.sh ${CMAKE_BINARY_DIR} test ${TEST_PATH})
@@ -328,8 +332,12 @@ endif()
 include(ThirdpartyToolchain)
 
 # Thrift requires these definitions for some types that we use
-add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETINET_IN_H -DHAVE_NETDB_H)
-add_definitions(-fPIC)
+add_definitions(-DHAVE_INTTYPES_H -DHAVE_NETDB_H)
+if (MSVC)
+  add_definitions(-DNOMINMAX)
+else()
+  add_definitions(-DHAVE_NETINET_IN_H -fPIC)
+endif()
 
 #############################################################
 # Compiler flags and release types
@@ -535,8 +543,10 @@ set(THRIFT_SRCS
   src/parquet/parquet_constants.cpp
   src/parquet/parquet_types.cpp)
 
-set_source_files_properties(src/parquet/parquet_types.cpp PROPERTIES
-  COMPILE_FLAGS -Wno-unused-variable)
+if (NOT MSVC)
+  set_source_files_properties(src/parquet/parquet_types.cpp PROPERTIES
+    COMPILE_FLAGS -Wno-unused-variable)
+endif()
 
 # List of thrift output targets
 set(THRIFT_OUTPUT_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src/parquet)
@@ -598,15 +608,11 @@ set(BUNDLED_STATIC_LIBS
   zlibstatic
 )
 
-add_library(parquet_objlib OBJECT
-  ${LIBPARQUET_SRCS}
-)
-
 # # Ensure that thrift compilation is done before using its generated headers
 # # in parquet code.
 add_custom_target(thrift-deps ALL
   DEPENDS ${THRIFT_OUTPUT_FILES})
-add_dependencies(parquet_objlib thrift-deps)
+set(PARQUET_DEPENDENCIES ${PARQUET_DEPENDENCIES} thrift-deps)
 
 if (NOT PARQUET_MINIMAL_DEPENDENCY)
 # These are libraries that we will link privately with parquet_shared (as they
@@ -620,65 +626,38 @@ if (NOT PARQUET_MINIMAL_DEPENDENCY)
   )
 # Although we don't link parquet_objlib against anything, we need it to depend
 # on these libs as we may generate their headers via ExternalProject_Add
-  add_dependencies(parquet_objlib
-    ${LIBPARQUET_INTERFACE_LINK_LIBS})
+  set(PARQUET_DEPENDENCIES ${PARQUET_DEPENDENCIES} ${LIBPARQUET_INTERFACE_LINK_LIBS})
 endif()
 
-set_property(TARGET parquet_objlib PROPERTY POSITION_INDEPENDENT_CODE 1)
-
-if(APPLE)
-  set(SHARED_LINK_FLAGS "-undefined dynamic_lookup")
-elseif()
+if(NOT APPLE AND NOT MSVC)
   # Localize thirdparty symbols using a linker version script. This hides them
   # from the client application. The OS X linker does not support the
   # version-script option.
   set(SHARED_LINK_FLAGS "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/src/parquet/symbols.map")
 endif()
 
-if (PARQUET_BUILD_SHARED)
-    add_library(parquet_shared SHARED $<TARGET_OBJECTS:parquet_objlib>)
-    set_target_properties(parquet_shared
-      PROPERTIES
-      LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
-      LINK_FLAGS "${SHARED_LINK_FLAGS}"
-      OUTPUT_NAME "parquet"
-      VERSION "${PARQUET_ABI_VERSION}"
-      SOVERSION "${PARQUET_SO_VERSION}")
-    target_link_libraries(parquet_shared
-      LINK_PRIVATE ${LIBPARQUET_INTERFACE_LINK_LIBS})
-    if (PARQUET_RPATH_ORIGIN)
-        if (APPLE)
-            set(_lib_install_rpath "@loader_path")
-        else()
-            set(_lib_install_rpath "\$ORIGIN")
-        endif()
-        set_target_properties(parquet_shared PROPERTIES
-            INSTALL_RPATH ${_lib_install_rpath})
-    elseif (APPLE)
-      set_target_properties(parquet_shared
-        PROPERTIES
-        BUILD_WITH_INSTALL_RPATH ON
-        INSTALL_NAME_DIR "@rpath")
-    endif()
-endif()
-
-if (PARQUET_BUILD_STATIC)
-    add_library(parquet_static STATIC $<TARGET_OBJECTS:parquet_objlib>)
-    set_target_properties(parquet_static
-      PROPERTIES
-      LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}"
-      OUTPUT_NAME "parquet")
-    target_link_libraries(parquet_static
-      LINK_PUBLIC ${LIBPARQUET_INTERFACE_LINK_LIBS})
-endif()
+ADD_LIB(parquet
+        SOURCES ${LIBPARQUET_SRCS}
+        LIB_BUILD_SHARED ${PARQUET_BUILD_SHARED}
+        LIB_BUILD_STATIC ${PARQUET_BUILD_STATIC}
+        DEPENDENCIES ${PARQUET_DEPENDENCIES}
+        SHARED_LINK_FLAGS ${SHARED_LINK_FLAGS}
+        SHARED_PRIVATE_LINK_LIBS ${LIBPARQUET_INTERFACE_LINK_LIBS}
+        STATIC_LINK_LIBS ${LIBPARQUET_INTERFACE_LINK_LIBS}
+        ABI_VERSION ${PARQUET_ABI_VERSION}
+        SO_VERSION ${PARQUET_SO_VERSION}
+        LIB_RPATH_ORIGIN ${PARQUET_RPATH_ORIGIN}
+)
 
 add_subdirectory(src/parquet)
 add_subdirectory(src/parquet/api)
 add_subdirectory(src/parquet/column)
 add_subdirectory(src/parquet/file)
 add_subdirectory(src/parquet/util)
 
-add_subdirectory(benchmarks)
+if (NOT MSVC)
+  add_subdirectory(benchmarks)
+endif()
 add_subdirectory(examples)
 add_subdirectory(tools)
 
@@ -691,16 +670,3 @@ add_custom_target(clean-all
    COMMAND ${CMAKE_BUILD_TOOL} clean
    COMMAND ${CMAKE_COMMAND} -P cmake_modules/clean-all.cmake
 )
-
-# installation
-
-if (PARQUET_BUILD_STATIC)
-    install(TARGETS parquet_static
-      ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-      LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()
-if (PARQUET_BUILD_SHARED)
-    install(TARGETS parquet_shared
-      ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-      LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR})
-endif()

LICENSE.txt

@@ -263,3 +263,14 @@ This product includes code from the Google styleguide.
 Copyright: 2009 Google Inc. All rights reserved.
 Homepage: https://github.com/google/styleguide
 License: 3-clause BSD
+
+--------------------------------------------------------------------------------
+
+This product includes code from the Google snappy.
+
+* cmake_modules/SnappyCMakeLists.txt is based on the code from the Google snappy.
+* cmake_modules/SnappyConfig.h is based on the code from the Google snappy.
+
+Copyright: 2009 Google Inc. All rights reserved.
+Homepage: https://github.com/google/snappy
+License: 3-clause BSD

README.md

@@ -56,6 +56,10 @@ which is required for Thrift:
 brew install boost
 ```
 
+### Windows
+
+Check [Windows developer guide][1] for instructions to build parquet-cpp on Windows.
+
 ## Third Party Dependencies
 
 - Apache Arrow (memory management, built-in IO, optional Array adapters)
@@ -265,3 +269,5 @@ coveralls -t $PARQUET_CPP_COVERAGE_TOKEN --gcov-options '\-l' -r $PARQUET_ROOT -
 
 Note that `gcov` throws off artifacts from the STL, so I excluded my toolchain
 root stored in `$NATIVE_TOOLCHAIN` to avoid a cluttered coverage report.
+
+[1]: https://github.com/apache/parquet-cpp/blob/master/docs/Windows.md

appveyor.yml

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Operating system (build VM template)
+os: Visual Studio 2015
+
+environment:
+  matrix:
+    - GENERATOR: NMake Makefiles
+      PYTHON: "3.5"
+      ARCH: "64"
+  MSVC_DEFAULT_OPTIONS: ON
+  BOOST_ROOT: C:\Libraries\boost_1_63_0
+  BOOST_LIBRARYDIR: C:\Libraries\boost_1_63_0\lib64-msvc-14.0
+
+init:
+  - set MINICONDA=C:\Miniconda35-x64
+  - set PATH=%MINICONDA%;%MINICONDA%/Scripts;%MINICONDA%/Library/bin;%PATH%
+  - if "%GENERATOR%"=="NMake Makefiles" call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall.bat" x64
+
+build_script:
+  - call ci\msvc-build.bat

benchmarks/decode_benchmark.cc

@@ -195,9 +195,9 @@ uint64_t TestPlainIntEncoding(const uint8_t* data, int num_values, int batch_siz
   uint64_t result = 0;
   parquet::PlainDecoder<parquet::Int64Type> decoder(nullptr);
   decoder.SetData(num_values, data, num_values * sizeof(int64_t));
-  int64_t values[batch_size];
+  std::vector<int64_t> values(batch_size);
   for (int i = 0; i < num_values;) {
-    int n = decoder.Decode(values, batch_size);
+    int n = decoder.Decode(values.data(), batch_size);
     for (int j = 0; j < n; ++j) {
       result += values[j];
     }
@@ -247,13 +247,13 @@ uint64_t TestBinaryPackedEncoding(const char* name, const std::vector<int64_t>&
     printf("  Encoded len: %d (%0.2f%%)\n", len, len * 100 / static_cast<float>(raw_len));
 
     uint64_t result = 0;
-    int64_t buf[benchmark_batch_size];
+    std::vector<int64_t> buf(benchmark_batch_size);
     parquet::StopWatch sw;
     sw.Start();
     for (int k = 0; k < benchmark_iters; ++k) {
       decoder.SetData(encoder.num_values(), buffer, len);
       for (size_t i = 0; i < values.size();) {
-        int n = decoder.Decode(buf, benchmark_batch_size);
+        int n = decoder.Decode(buf.data(), benchmark_batch_size);
         for (int j = 0; j < n; ++j) {
           result += buf[j];
         }

ci/msvc-build.bat

@@ -0,0 +1,27 @@
+@rem Licensed to the Apache Software Foundation (ASF) under one
+@rem or more contributor license agreements.  See the NOTICE file
+@rem distributed with this work for additional information
+@rem regarding copyright ownership.  The ASF licenses this file
+@rem to you under the Apache License, Version 2.0 (the
+@rem "License"); you may not use this file except in compliance
+@rem with the License.  You may obtain a copy of the License at
+@rem
+@rem   http://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing,
+@rem software distributed under the License is distributed on an
+@rem "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+@rem KIND, either express or implied.  See the License for the
+@rem specific language governing permissions and limitations
+@rem under the License.
+
+@echo on
+
+mkdir build
+cd build
+
+cmake -G "%GENERATOR%" ^
+      -DCMAKE_BUILD_TYPE=Release ^
+      .. || exit /B
+
+nmake || exit /B