diff --git a/.github/workflows/emscripten.yml b/.github/workflows/emscripten.yml new file mode 100644 index 0000000..6f1600c --- /dev/null +++ b/.github/workflows/emscripten.yml @@ -0,0 +1,68 @@ +# Build das2C for WebAssembly with buildfiles/emscripten.mak on Ubuntu 24.04 +# and run the wasm test programs under node. Regression testing only: +# nothing is uploaded, the wasm archive is not a release file. das2js is +# the downstream that links it into a browser module. release.yml still +# calls this workflow and publishes nothing unless it passes. +# +# The wasm output doesn't depend on the host, so the runner OS only decides +# how the toolchain gets installed. Pinned like ubuntu.yml, so the build +# changes only when this file does. +# +# emsdk is cloned and installed directly rather than through a marketplace +# action, the same steps notes/build_for_emscripten.md gives a developer. +# EMSDK_VER pins emscripten itself; bump it here to move CI to a new +# compiler. emsdk_env.sh sets PATH and EMSDK_NODE for one shell only, so +# every step that needs emcc or node sources it. +# +# Nothing is cached. emscripten.mak downloads and builds expat and FFTW on +# every run, which is what keeps the pinned URLs and SHA-256 sums honest: +# a release that moves or changes upstream fails here first. FFTW is the +# slow part of the run. + +name: Emscripten + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + workflow_dispatch: + workflow_call: + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-24.04 + + env: + EMSDK_VER: 6.0.13 + + defaults: + run: + shell: bash + + steps: + - uses: actions/checkout@v7 + + - name: Install emsdk + run: | + git clone --depth 1 https://github.com/emscripten-core/emsdk.git "$RUNNER_TEMP/emsdk" + cd "$RUNNER_TEMP/emsdk" + ./emsdk install "$EMSDK_VER" + ./emsdk activate "$EMSDK_VER" + + # Separate from the test step, so a failure says whether it was the + # dependency downloads, the compile or the link. -j reaches the expat + # and FFTW builds too, see the expat rule in emscripten.mak. + - name: Build + run: | + source "$RUNNER_TEMP/emsdk/emsdk_env.sh" + emcc --version + make -j"$(nproc)" -f buildfiles/emscripten.mak build_test + + - name: Test + run: | + source "$RUNNER_TEMP/emsdk/emsdk_env.sh" + make -f buildfiles/emscripten.mak test diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a3eb672..88c7fc1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,7 +12,8 @@ # das2C--macos-arm64.tar.gz The same for Apple Silicon # # The builds themselves are windows.yml, ubuntu.yml and macos.yml, which -# also run on every push to master. +# also run on every push to master. emscripten.yml runs too, though it +# ships nothing: a release waits on the wasm tests like the others. # A tag with a '-' in it, such as v3.0-pre8, makes a pre-release. Any other # tag must match "version-string" in vcpkg.json, so a release can't go out # with a mismatched version number. Nothing is published unless every build @@ -78,8 +79,12 @@ jobs: with: version: ${{ needs.check.outputs.version }} + emscripten: + needs: check + uses: ./.github/workflows/emscripten.yml + publish: - needs: [ check, windows, ubuntu, macos ] + needs: [ check, windows, ubuntu, macos, emscripten ] runs-on: ubuntu-latest permissions: contents: write diff --git a/.gitignore b/.gitignore index 969705e..f387b41 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ build.*/ +deps.*/ build/ *.obj buildfiles/nbproject/private diff --git a/CMakeLists.txt b/CMakeLists.txt index 1ace938..291c99a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ # cmake --install build.cmake/default --prefix C:\opt # cmake --build --preset default --target uninstall # -# On Windows the programs run on das3.dll, as with Windows.mak. Elsewhere +# On Windows the programs run on das3.dll, as with windows.mak. Elsewhere # they link the static library, use the "dll" preset to add libdas3.so. # Without presets, pass # -DCMAKE_TOOLCHAIN_FILE=%VCPKG_ROOT%/scripts/buildsystems/vcpkg.cmake, or @@ -74,7 +74,7 @@ if(CMAKE_HOST_WIN32 AND CMAKE_GENERATOR MATCHES "Ninja" AND NOT DEFINED CMAKE_C_ endif() # Static vcpkg dependencies built for the DLL C runtime (/MD) by default on -# Windows, the same as Windows.mak, so they all land in das3.dll. Must be set +# Windows, the same as windows.mak, so they all land in das3.dll. Must be set # before project(). The C runtime is CMake's default, /MD, except with a # fully static triplet such as x64-windows-static, see below. Without vcpkg, # as under conda, /MD matches the dependency DLLs. @@ -94,7 +94,7 @@ include(CMakePackageConfigHelpers) option(DAS_WITH_SPICE "Build the SPICE programs" ON) option(DAS_WITH_CDF "Build the CDF programs" ON) -# On Windows the programs run on das3.dll, as with Windows.mak, so that the +# On Windows the programs run on das3.dll, as with windows.mak, so that the # dependencies are in the DLL once instead of in every program. Elsewhere # they link the static library, as with Linux.mak. if(WIN32) @@ -179,7 +179,7 @@ endif() # The makefiles keep assert() active in optimized builds and so do we. For # MSVC also put debug info in the objects (/Z7) so libdas3.lib carries it, -# leave inlining at the /O2 level as Windows.mak does, and skip incremental +# leave inlining at the /O2 level as windows.mak does, and skip incremental # linking, which leaves 20 MB+ .ilk files. Linking with /debug turns off # removal of unused code, so optimized builds ask for it back (/OPT). foreach(_cfg RELEASE RELWITHDEBINFO MINSIZEREL) diff --git a/Makefile b/Makefile index a13e2c9..49e0572 100644 --- a/Makefile +++ b/Makefile @@ -20,12 +20,6 @@ ifeq ($(INST_DOC),) INST_DOC=$(INST_SHARE)/doc endif -# Cross compiling to WebAssembly: objects land in build.wasm32 and the -# emscripten rules below replace the host platform's -ifeq ($(EMSCRIPTEN),yes) -N_ARCH=wasm32 -endif - ifeq ($(N_ARCH),) N_ARCH=/ #N_ARCH=$(shell uname).$(shell uname -m) @@ -46,10 +40,8 @@ endif # SPICE and CDF programs are part of the standard build, SPICE=no or CDF=no # leaves them out (so does any value but "yes"). -ifneq ($(EMSCRIPTEN),yes) SPICE?=yes CDF?=yes -endif BLD_CSPICE=0 ifeq ($(SPICE),yes) @@ -79,6 +71,11 @@ C_HDR_DIR:=$(CURDIR) BUILD_DIR:=build.$(N_ARCH) C_BUILD_DIR:=$(CURDIR)/$(BUILD_DIR) +# Downloaded sources and the dependency builds made from them (CDF, CSPICE). +# Beside the build directory rather than in it, so clean can remove all of +# $(BUILD_DIR) and only distclean costs a re-download. +DEPS_DIR:=deps.$(N_ARCH) + ############################################################################## # Native Platform specific include @@ -100,9 +97,14 @@ export INST_NAT_LIB export C_BUILD_DIR export C_HDR_DIR -ifeq ($(EMSCRIPTEN),yes) -$(info Cross compiling for wasm32 with emscripten) -include buildfiles/Emscripten.mak -else +# The WebAssembly build is not selected here, use: +# +# make -f buildfiles/emscripten.mak +# +# The windows build is not selected here either, use: +# +# nmake /f buildfiles\windows.mak +# -or- +# cmake --preset default + include buildfiles/$(UNAME).mak -endif diff --git a/README.md b/README.md index 719b6c6..5f02d57 100644 --- a/README.md +++ b/README.md @@ -98,15 +98,16 @@ For a WebAssembly build of the library alone (no utilities), with emscripten's emsdk activated in the shell: ```bash -$ make EMSCRIPTEN=yes EXPAT_INC=/path/to/expat/lib build -$ make EMSCRIPTEN=yes EXPAT_INC=/path/to/expat/lib EXPAT_LIB=/path/to/libexpat.a test +$ make -f buildfiles/emscripten.mak +$ make -f buildfiles/emscripten.mak test + +# To rebuild +$ make -f buildfiles/emscripten.mak clean # Removes only das2C output +$ make -f buildfiles/emscripten.mak distclean # Removes expat and FFTW as well ``` -Output lands in `build.wasm32/`. Emscripten provides zlib but not expat, so a -libexpat built for wasm32 has to be supplied. See -[notes/build_for_emscripten.md](notes/build_for_emscripten.md) for the -toolchain setup, what is left out of the archive and why, and how to link -the result into a program of your own. +Output lands in `build.wasm32/`. The build expat and fftw and builds them +for wasm32 under `deps.wasm32/`. See [notes/build_for_emscripten.md](notes/build_for_emscripten.md) for the toolchain setup. For Windows systems issue the following commands in a command shell to build and test the software. Dependencies are installed by [vcpkg](https://vcpkg.io) @@ -117,8 +118,8 @@ See [notes/build_for_windows.md](notes/build_for_windows.md) for details. > vcvars64.bat ::puts nmake.exe, cl.exe, link.exe, etc. on your path > set VCPKG_ROOT= ::location of your vcpkg clone -> nmake /f buildfiles\Windows.mak SPICE=yes CDF=yes -> nmake /f buildfiles\Windows.mak SPICE=yes CDF=yes test +> nmake /f buildfiles\windows.mak +> nmake /f buildfiles\windows.mak test ``` On Windows you can also build with [CMake](https://cmake.org), which Visual @@ -128,9 +129,8 @@ tested on Windows so far. ```batchfile > vcvars64.bat -> set VCPKG_ROOT= ::location of your vcpkg clone - -> cmake --preset default ::configure, vcpkg builds the dependencies +> set VCPKG_ROOT= ::location of your vcpkg clone +> cmake --preset default ::configure step, vcpkg builds the dependencies > cmake --build --preset default > ctest --preset default ``` @@ -149,10 +149,10 @@ $ make PREFIX=/usr/local install # adjust destination to taste ``` or on Windows: ```batch -nmake /f buildfiles\Windows.mak PREFIX=C:\local install +nmake /f buildfiles\windows.mak PREFIX=C:\opt install ``` ```batch -cmake --install build.cmake\default --prefix C:\local +cmake --install build.cmake\default --prefix C:\opt ``` > **Side note:** a CMake install can be undone with diff --git a/buildfiles/Android.mak b/buildfiles/Android.mak index 6c8a3e6..fe598a5 100644 --- a/buildfiles/Android.mak +++ b/buildfiles/Android.mak @@ -25,6 +25,12 @@ TEST_PROGS=TestUnits TestArray TestDs TestBuilder \ TestV3Read TestProp TestIter TestUri TestFilter TestValue TestRaggedEncode \ TestGen TestForm TestCplx TestVar TestVarSubset TestDim TestDatum +# Set by the top level Makefile, which includes this file. Read on its own +# (make -f) every output path would start at /, so refuse. +ifeq ($(BUILD_DIR),) +$(error buildfiles/Android.mak is included by Makefile, run make from the top of the source tree) +endif + BD=$(BUILD_DIR) ############################################################################## diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index 96104d5..494be7b 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -19,10 +19,13 @@ # tools. # # CSPICE: From CSPICE_INC and CSPICE_LIB when set, otherwise the NAIF -# toolkit for this machine's architecture is downloaded to $(BD)/cspice +# toolkit for this machine's architecture is downloaded to +# $(DEPS_DIR)/cspice # # CDF: From CDF_INC and CDF_LIB when set, otherwise the SPDF source -# distribution is downloaded and built under $(BD) +# distribution is downloaded and built under $(DEPS_DIR) +# +# Only distclean removes $(DEPS_DIR), clean leaves the downloads alone. # # Apple's clang honors MACOSX_DEPLOYMENT_TARGET from the environment, set it # to build for older MacOS versions than the one running the build. Static @@ -32,6 +35,12 @@ MD5SUM=md5 -r export MD5SUM +# Set by the top level Makefile, which includes this file. Read on its own +# (make -f) every output path would start at /, so refuse. +ifeq ($(BUILD_DIR),) +$(error buildfiles/Darwin.mak is included by Makefile, run make from the top of the source tree) +endif + BD=$(BUILD_DIR) ############################################################################## @@ -52,8 +61,8 @@ LOC_CDF_ENV:=x86_64 endif ifeq ($(BLD_CDF),1) -CDF_INC:=$(BD)/$(LOC_CDF_DIST)/src/include -CDF_LIB:=$(BD)/$(LOC_CDF_DIST)/src/lib/libcdf.a +CDF_INC:=$(DEPS_DIR)/$(LOC_CDF_DIST)/src/include +CDF_LIB:=$(DEPS_DIR)/$(LOC_CDF_DIST)/src/lib/libcdf.a endif # Downstream packages (das2py) take libcdf.dylib from $(BD) so that they run @@ -66,8 +75,8 @@ CDF_SO_REDIST:=$(BD)/libcdf.dylib endif ifeq ($(BLD_CSPICE),1) -CSPICE_INC:=./$(BD)/cspice/include -CSPICE_LIB:=./$(BD)/cspice/lib/cspice.a +CSPICE_INC:=./$(DEPS_DIR)/cspice/include +CSPICE_LIB:=./$(DEPS_DIR)/cspice/lib/cspice.a endif @@ -76,7 +85,7 @@ endif TARG=libdas3 -# File lists are shared with Linux.mak and Windows.mak, add new sources there +# File lists are shared with Linux.mak and windows.mak, add new sources there include buildfiles/sources.mak SRCS:=$(notdir $(LIB_SRCS)) @@ -239,6 +248,10 @@ $(BD): @if [ ! -e "$(BD)" ]; then echo mkdir $(BD); \ mkdir $(BD); chmod g+w $(BD); fi +$(DEPS_DIR): + @if [ ! -e "$(DEPS_DIR)" ]; then echo mkdir $(DEPS_DIR); \ + mkdir $(DEPS_DIR); chmod g+w $(DEPS_DIR); fi + # Records which objects make up the library. A source dropped from the list # changes that set without making any object newer than the archive, so make # alone would keep a stale one. The file is rewritten only when the set @@ -301,8 +314,8 @@ endif build_dep:$(_DEPS) -$(CSPICE_LIB): $(BD)/cspice.tar - cd $(BD) && tar -mxf cspice.tar +$(CSPICE_LIB): $(DEPS_DIR)/cspice.tar + cd $(DEPS_DIR) && tar -mxf cspice.tar # Checked on every build, see the same rule in Linux.mak for why ifeq ($(BLD_CDF),1) @@ -321,20 +334,20 @@ $(CDF_SO_REDIST): FORCE | $(BD) echo "cp $(CDF_SO) $@"; cp "$(CDF_SO)" $@; \ fi -$(BD)/cspice.tar: | $(BD) - curl -f -L $(LOC_CSPICE_URL) > $(BD)/cspice.tar.Z - uncompress $(BD)/cspice.tar.Z +$(DEPS_DIR)/cspice.tar: | $(DEPS_DIR) + curl -f -L $(LOC_CSPICE_URL) > $(DEPS_DIR)/cspice.tar.Z + uncompress $(DEPS_DIR)/cspice.tar.Z # CDF's own tools need curses, the library does not. Without a full Xcode # install the CDF makefile prints xcodebuild errors while probing for an Xcode # version, these are harmless. -$(CDF_LIB): $(BD)/$(LOC_CDF_DIST) - cd $(BUILD_DIR)/$(LOC_CDF_DIST) && $(MAKE) OS=macosx ENV=$(LOC_CDF_ENV) \ +$(CDF_LIB): $(DEPS_DIR)/$(LOC_CDF_DIST) + cd $(DEPS_DIR)/$(LOC_CDF_DIST) && $(MAKE) OS=macosx ENV=$(LOC_CDF_ENV) \ CURSES=no FORTRAN=no all -$(BD)/$(LOC_CDF_DIST): | $(BD) - curl -f -L $(LOC_CDF_URL) > $(BD)/$(LOC_CDF_DIST)-cdf.tar.gz - cd $(BD) && tar -xf $(LOC_CDF_DIST)-cdf.tar.gz +$(DEPS_DIR)/$(LOC_CDF_DIST): | $(DEPS_DIR) + curl -f -L $(LOC_CDF_URL) > $(DEPS_DIR)/$(LOC_CDF_DIST)-cdf.tar.gz + cd $(DEPS_DIR) && tar -xf $(LOC_CDF_DIST)-cdf.tar.gz # Run tests @@ -529,13 +542,12 @@ $(INST_DOC)/das2C:$(BD)/html # Cleanup #################################################################### -distclean: +# Same split as Linux.mak: all of $(BD) is ours, $(DEPS_DIR) costs a re-fetch +clean: if [ -d "$(BD)" ] ; then rm -r $(BD) ; fi -clean: - -rm $(BD)/das* $(BD)/Test* $(BD)/libdas* - -rm $(BD)/*.o $(BD)/*.d $(BD)/ex* - -rm $(BD)/cred_test* +distclean: clean + if [ -d "$(DEPS_DIR)" ] ; then rm -r $(DEPS_DIR) ; fi ## Automatic dependency tree generation for C code ########################### diff --git a/buildfiles/Emscripten.mak b/buildfiles/Emscripten.mak deleted file mode 100644 index f27d636..0000000 --- a/buildfiles/Emscripten.mak +++ /dev/null @@ -1,246 +0,0 @@ -############################################################################## -# WebAssembly cross build via emscripten -# -# Selected by the top level Makefile when EMSCRIPTEN=yes: -# -# make EMSCRIPTEN=yes EXPAT_INC=/path/to/expat/lib build -# make EMSCRIPTEN=yes EXPAT_INC=... EXPAT_LIB=/path/to/libexpat.a test -# -# Builds the library archive and, for 'test', a subset of the test programs -# as node.js scripts. No utilities, no shared object, no program install. -# The archive is meant to be linked into a wasm module by a downstream -# project that owns the JavaScript side (das2js). -# -# Prerequisites: -# -# emsdk activated in the calling shell so emcc and emar are on PATH. The -# bundled node is found through EMSDK_NODE, which emsdk_env.sh exports. -# -# EXPAT_INC directory holding expat.h. Emscripten ships a zlib port but -# no expat port, so the caller supplies one built for wasm32. -# EXPAT_LIB the matching libexpat.a, needed only to link test programs. -# -# What is left out, and why: -# -# http.c credentials.c catalog.c sockets, OpenSSL and the node cache do -# not exist in a browser. fetch() on the JS -# side drives DasNodeWalk instead; node.c, -# source.c and req.c never touch a transport -# dft.c FFTW is not built for wasm yet. DFT=yes -# puts it back given FFTW_INC. -# cli.c send.c program scaffolding, nothing a library -# consumer needs -# -# Dropping those files leaves dangling calls in util.c, io.c and buffer.c. -# Two feature macros close them: DAS_NO_NETWORK compiles out the socket and -# SSL stream modes, DAS_NO_DFT skips FFTW setup in das_init(). They are -# defined here and nowhere else, so every native build is unaffected. - -# Cleaning needs neither the compiler nor the expat location -ifeq ($(filter clean distclean,$(MAKECMDGOALS)),) -ifeq ($(shell command -v emcc 2>/dev/null),) -$(error emcc not found on PATH. Activate emsdk first: source /path/to/emsdk/emsdk_env.sh) -endif -endif - -BD=$(BUILD_DIR) - -############################################################################## -# Project definitions - -TARG=libdas3 - -include buildfiles/sources.mak - -NET_SRCS:=das3/http.c das3/credentials.c das3/catalog.c -PROG_SRCS:=das3/cli.c das3/send.c -DFT_SRCS:=das3/dft.c - -WASM_SRCS:=$(filter-out $(NET_SRCS) $(PROG_SRCS),$(LIB_SRCS)) -FEATURE_DEFS:=-DDAS_NO_NETWORK - -ifeq ($(DFT),yes) -ifeq ($(FFTW_INC),) -$(error DFT=yes needs FFTW_INC, the directory holding fftw3.h built for wasm32) -endif -DEP_INC:=-I$(FFTW_INC) -else -WASM_SRCS:=$(filter-out $(DFT_SRCS),$(WASM_SRCS)) -FEATURE_DEFS:=$(FEATURE_DEFS) -DDAS_NO_DFT -endif - -ifeq ($(filter clean distclean,$(MAKECMDGOALS)),) -ifeq ($(EXPAT_INC),) -$(error EXPAT_INC is not set. Point it at a directory holding expat.h built for wasm32) -endif -endif -DEP_INC:=$(DEP_INC) -I$(EXPAT_INC) - -SRCS:=$(notdir $(WASM_SRCS)) -HDRS:=$(notdir $(LIB_HDRS)) - -# Test programs that need nothing from the excluded sources. Anything that -# opens a URL, reads credentials, or needs SPICE stays out. -TEST_PROGS:=TestValue TestDatum TestGen TestForm TestCplx TestVar \ - TestVarSubset TestDim TestDesc TestUnits TestTT2000 TestArray TestDs TestIter \ - TestRaggedEncode TestFilter TestUri TestV3Read TestFeed TestWalk - -############################################################################## -# Build definitions - -CC=emcc -AR=emar - -# emcc is clang: the gcc-only -Wno-stringop-truncation of Linux.mak would -# itself be an error here -WARNINGS:=-Wall -Werror -Wno-deprecated-declarations - -DEFINES:=-D_XOPEN_SOURCE=600 $(FEATURE_DEFS) - -OPTFLAGS=-O2 -g - -# --use-port=zlib supplies zlib.h at compile time and the library at link -# time. -MMD -MP writes a .d beside each .o for header dependency tracking, -# same scheme as Linux.mak. -CFLAGS=$(WARNINGS) $(OPTFLAGS) $(DEFINES) -std=c99 -I. $(DEP_INC) \ - --use-port=zlib -MMD -MP - -# Test programs run under node with the real filesystem mounted (NODERAWFS) -# so fixtures need no embedding. Flag notes: -# * EXIT_RUNTIME makes main()'s return value the process exit status -# * The 1 MiB stack is native-like; emscripten defaults to 64 KiB. -# * -g keeps DWARF, which makes emcc warn that wasm-opt ran fewer passes -# with -Werror on the same command line that would fail the link. -# -TEST_LFLAGS=--use-port=zlib -sENVIRONMENT=node -sNODERAWFS=1 -sEXIT_RUNTIME=1 \ - -sALLOW_MEMORY_GROWTH=1 -sSTACK_SIZE=1048576 \ - -Wno-limited-postlink-optimizations - -ifeq ($(NODE),) -ifeq ($(EMSDK_NODE),) -NODE:=node -else -NODE:=$(EMSDK_NODE) -endif -endif - -############################################################################## -# Derived definitions - -BUILD_OBJS=$(patsubst %.c,$(BD)/%.o,$(SRCS)) - -BUILD_TEST_PROGS=$(patsubst %,$(BD)/%.js,$(TEST_PROGS)) - -INST_HDRS=$(patsubst %.h,$(DESTDIR)$(INST_INC)/das3/%.h,$(HDRS)) - -# Fixture lists mirror Linux.mak, see the comment there for the exclusions -V3_FIXTURES := \ - $(filter-out examples/ex28_%,$(wildcard examples/*.d3b examples/*.d3t)) \ - $(filter-out test/streams/notimp_% test/streams/reject_%,$(wildcard test/streams/*.d3b test/streams/*.d3t)) - -V2_FIXTURES := $(wildcard test/streams/*.d2s test/streams/*.d2t examples/*.d2s examples/*.d2t) - -############################################################################## -# Pattern rules - -$(BD)/%.o:das3/%.c | $(BD) - $(CC) -c $(CFLAGS) -o $@ $< - -# emcc picks the output kind from the suffix: a .js loader with the .wasm -# beside it -$(BD)/%.js:test/%.c $(BD)/$(TARG).a | $(BD) - @if [ -z "$(EXPAT_LIB)" ]; then echo "ERROR: EXPAT_LIB not set, needed to link test programs"; exit 3; fi - $(CC) $(CFLAGS) $< $(BD)/$(TARG).a $(EXPAT_LIB) $(TEST_LFLAGS) -o $@ - -$(DESTDIR)$(INST_NAT_LIB)/%.a:$(BD)/%.a - install -D -m 664 $< $@ - -$(DESTDIR)$(INST_INC)/das3/%.h:das3/%.h - install -D -m 664 $< $@ - -# No .SECONDARY here, unlike Linux.mak: nothing is an intermediate. Objects -# are explicit prereqs of the archive and the test .js files link straight from -# test/*.c. -.PHONY: build test install dev_install clean distclean - -############################################################################## -# Explicit rules - -build: $(BD)/$(TARG).a - -$(BD): - @if [ ! -e "$(BD)" ]; then echo mkdir $(BD); mkdir $(BD); chmod g+w $(BD); fi - -# Records which objects make up the library. DFT= changes that set without -# making any object newer than the archive, so make alone would keep a stale -# one. The file is rewritten only when the set differs. -$(BD)/lib_members: FORCE | $(BD) - @echo '$(BUILD_OBJS)' | cmp -s - $@ || echo '$(BUILD_OBJS)' > $@ - -FORCE: -.PHONY: FORCE - -# The member set depends on DFT, so start the archive fresh rather than let -# ar keep stale objects from an earlier configuration -$(BD)/$(TARG).a:$(BUILD_OBJS) $(BD)/lib_members | $(BD) - rm -f $@ - $(AR) rc $@ $(BUILD_OBJS) - -test: $(BD)/$(TARG).a $(BUILD_TEST_PROGS) - @echo "INFO: wasm test programs run under $(NODE)" - @echo "INFO: Running unit test for the value layer, $(BD)/TestValue..." - @$(NODE) $(BD)/TestValue.js - @echo "INFO: Running unit test for the datum layer, $(BD)/TestDatum..." - @$(NODE) $(BD)/TestDatum.js - @echo "INFO: Running unit test for the DasGen value sources, $(BD)/TestGen..." - @$(NODE) $(BD)/TestGen.js - @echo "INFO: Running unit test for the DasForm formalisms, $(BD)/TestForm..." - @$(NODE) $(BD)/TestForm.js - @echo "INFO: Running unit test for complex arithmetic, $(BD)/TestCplx..." - @$(NODE) $(BD)/TestCplx.js - @echo "INFO: Running unit test for the DasVar layer, $(BD)/TestVar..." - @$(NODE) $(BD)/TestVar.js - @echo "INFO: Running unit test for DasVar bulk reads, $(BD)/TestVarSubset..." - @$(NODE) $(BD)/TestVarSubset.js - @echo "INFO: Running unit test for the DasDim container, $(BD)/TestDim..." - @$(NODE) $(BD)/TestDim.js - @echo "INFO: Running unit test for descriptor property storage, $(BD)/TestDesc..." - @$(NODE) $(BD)/TestDesc.js - @echo "INFO: Running unit test to test units, $(BD)/TestUnits..." - @$(NODE) $(BD)/TestUnits.js - @echo "INFO: Running unit test for TT2000 leap seconds, $(BD)/TestTT2000..." - @$(NODE) $(BD)/TestTT2000.js - @echo "INFO: Running unit test for dynamic arrays, $(BD)/TestArray..." - @$(NODE) $(BD)/TestArray.js - @echo "INFO: Running unit test for dataset shape/length merge, $(BD)/TestDs..." - @$(NODE) $(BD)/TestDs.js - @echo "INFO: Running unit test for stream parsing over all fixtures, $(BD)/TestV3Read..." - $(NODE) $(BD)/TestV3Read.js test/streams/tag_test.dNt $(V2_FIXTURES) $(V3_FIXTURES) - @echo "INFO: Running unit test for push parsing vs. pull parsing, $(BD)/TestFeed..." - $(NODE) $(BD)/TestFeed.js test/streams/tag_test.dNt $(V2_FIXTURES) $(V3_FIXTURES) - @echo "INFO: Running unit test for ragged and unique iteration, $(BD)/TestIter..." - $(NODE) $(BD)/TestIter.js - @echo "INFO: Running unit test for ragged binary re-encode, $(BD)/TestRaggedEncode..." - $(NODE) $(BD)/TestRaggedEncode.js examples/ex30_cassini_ragged_notlast.d3b \ - examples/ex31_efi_ragged_vec.d3b examples/ex32_marsis_2d_ragged.d3b \ - examples/ex34_ragged_fixstr.d3b examples/ex38_wbr_wfrm_tags.d3b \ - examples/ex39_sandwich.d3b - @echo "INFO: Running unit test for filter dataset ops (copy/swap), $(BD)/TestFilter..." - $(NODE) $(BD)/TestFilter.js - $(NODE) $(BD)/TestUri.js $(BD)/uri_tplt - @echo "INFO: ===============================================" - @echo "INFO: All wasm test programs completed without errors" - @echo "INFO: ===============================================" - -# Only the archive and headers: there are no programs to install -dev_install: $(DESTDIR)$(INST_NAT_LIB)/$(TARG).a $(INST_HDRS) -install: dev_install - -distclean: - if [ -d "$(BD)" ] ; then rm -r $(BD) ; fi - -clean: - -rm $(BD)/*.o $(BD)/*.d $(BD)/*.js $(BD)/*.wasm $(BD)/libdas* - -# Header dependencies written by -MMD above --include $(wildcard $(BD)/*.d) diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 8bffa73..e406c80 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -3,6 +3,12 @@ MD5SUM=md5sum export MD5SUM +# Set by the top level Makefile, which includes this file. Read on its own +# (make -f) every output path would start at /, so refuse. +ifeq ($(BUILD_DIR),) +$(error buildfiles/Linux.mak is included by Makefile, run make from the top of the source tree) +endif + BD=$(BUILD_DIR) ############################################################################## @@ -11,15 +17,16 @@ BD=$(BUILD_DIR) # You may already have libcdf.a and cspice.a in your build tree for other # parts of your project. If so use them by setting CSPICE_INC, CSPICE_LIB, # CDF_INC and CDF_LIB. If not, you can issue make build_dep first and -# dependency sources will be downloaded and built. +# dependency sources will be downloaded and built in $(DEPS_DIR), which +# only distclean removes. LOC_CDF_DIST:=cdf39_1-dist LOC_CDF_URL:=https://spdf.gsfc.nasa.gov/pub/software/cdf/dist/cdf39_1/linux/$(LOC_CDF_DIST)-cdf.tar.gz LOC_CSPICE_URL:=https://naif.jpl.nasa.gov/pub/naif/toolkit//C/PC_Linux_GCC_64bit/packages/cspice.tar.Z ifeq ($(BLD_CDF),1) -CDF_INC:=$(BD)/$(LOC_CDF_DIST)/src/include -CDF_LIB:=$(BD)/$(LOC_CDF_DIST)/src/lib/libcdf.a +CDF_INC:=$(DEPS_DIR)/$(LOC_CDF_DIST)/src/include +CDF_LIB:=$(DEPS_DIR)/$(LOC_CDF_DIST)/src/lib/libcdf.a endif # Downstream packages (das2py, dasflex) take libcdf.so from $(BD) so that they @@ -32,8 +39,8 @@ CDF_SO_REDIST:=$(BD)/libcdf.so endif ifeq ($(BLD_CSPICE),1) -CSPICE_INC:=./$(BD)/cspice/include -CSPICE_LIB:=./$(BD)/cspice/lib/cspice.a +CSPICE_INC:=./$(DEPS_DIR)/cspice/include +CSPICE_LIB:=./$(DEPS_DIR)/cspice/lib/cspice.a endif @@ -42,7 +49,7 @@ endif TARG=libdas3 -# File lists are shared with Windows.mak, add new sources there +# File lists are shared with windows.mak, add new sources there include buildfiles/sources.mak SRCS:=$(notdir $(LIB_SRCS)) @@ -212,6 +219,10 @@ $(BD): @if [ ! -e "$(BD)" ]; then echo mkdir $(BD); \ mkdir $(BD); chmod g+w $(BD); fi +$(DEPS_DIR): + @if [ ! -e "$(DEPS_DIR)" ]; then echo mkdir $(DEPS_DIR); \ + mkdir $(DEPS_DIR); chmod g+w $(DEPS_DIR); fi + # Records which objects make up the library. A source dropped from the list # changes that set without making any object newer than the archive, so make # alone would keep a stale one. The file is rewritten only when the set @@ -278,8 +289,8 @@ endif build_dep:$(_DEPS) -$(CSPICE_LIB): $(BD)/cspice.tar - cd $(BD) && tar -mxvf cspice.tar +$(CSPICE_LIB): $(DEPS_DIR)/cspice.tar + cd $(DEPS_DIR) && tar -mxvf cspice.tar # Checked on every build, since CDF_LIB may now point at a different CDF than # the copy came from. The copy is rewritten only when the contents differ, so @@ -297,14 +308,14 @@ $(CDF_SO_REDIST): FORCE | $(BD) echo "cp $(CDF_SO) $@"; cp "$(CDF_SO)" $@; \ fi -$(BD)/cspice.tar: | $(BD) - curl -f -L $(LOC_CSPICE_URL) > $(BD)/cspice.tar.Z - uncompress $(BD)/cspice.tar.Z +$(DEPS_DIR)/cspice.tar: | $(DEPS_DIR) + curl -f -L $(LOC_CSPICE_URL) > $(DEPS_DIR)/cspice.tar.Z + uncompress $(DEPS_DIR)/cspice.tar.Z # CDF's own tools need curses, the library does not. Leaving them text-only # keeps ncurses headers off the list of things a default build requires. -$(CDF_LIB): $(BD)/$(LOC_CDF_DIST) - cd $(BUILD_DIR)/$(LOC_CDF_DIST) && $(MAKE) OS=linux ENV=gnu CURSES=no \ +$(CDF_LIB): $(DEPS_DIR)/$(LOC_CDF_DIST) + cd $(DEPS_DIR)/$(LOC_CDF_DIST) && $(MAKE) OS=linux ENV=gnu CURSES=no \ FORTRAN=no all # The shared library comes out of the same CDF build. Without saying so, @@ -314,9 +325,9 @@ ifeq ($(BLD_CDF),1) $(CDF_SO): $(CDF_LIB) ; endif -$(BD)/$(LOC_CDF_DIST): | $(BD) - curl -f -L $(LOC_CDF_URL) > $(BD)/$(LOC_CDF_DIST)-cdf.tar.gz - cd $(BD) && tar -xvf $(LOC_CDF_DIST)-cdf.tar.gz +$(DEPS_DIR)/$(LOC_CDF_DIST): | $(DEPS_DIR) + curl -f -L $(LOC_CDF_URL) > $(DEPS_DIR)/$(LOC_CDF_DIST)-cdf.tar.gz + cd $(DEPS_DIR) && tar -xvf $(LOC_CDF_DIST)-cdf.tar.gz # Run tests @@ -563,13 +574,14 @@ $(INST_DOC)/das2C:$(BD)/html # Cleanup #################################################################### -distclean: +# Everything in $(BD) is ours, so clean takes all of it. Downloads and the +# dependency builds live in $(DEPS_DIR) and cost a re-fetch, only distclean +# goes there. The libcdf.so copy in $(BD) comes back with the next build. +clean: if [ -d "$(BD)" ] ; then rm -r $(BD) ; fi -clean: - -rm $(BD)/das* $(BD)/Test* $(BD)/libdas* - -rm $(BD)/*.o $(BD)/ex* - -rm $(BD)/cred_test* +distclean: clean + if [ -d "$(DEPS_DIR)" ] ; then rm -r $(DEPS_DIR) ; fi ## Automatic dependency tree generation for C code ########################### diff --git a/buildfiles/SunOS.mak b/buildfiles/SunOS.mak index a3f34ea..7f36495 100644 --- a/buildfiles/SunOS.mak +++ b/buildfiles/SunOS.mak @@ -35,6 +35,12 @@ TEST_PROGS=TestUnits TestArray TestDs TestBuilder \ TestV3Read TestProp TestIter TestUri TestFilter TestValue TestRaggedEncode \ TestGen TestForm TestCplx TestVar TestVarSubset TestDim TestDatum +# Set by the top level Makefile, which includes this file. Read on its own +# (make -f) every output path would start at /, so refuse. +ifeq ($(BUILD_DIR),) +$(error buildfiles/SunOS.mak is included by Makefile, run make from the top of the source tree) +endif + BD=$(BUILD_DIR) ############################################################################## diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index e02ba14..01021f4 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -1,5 +1,5 @@ # CTest definitions, included by the top CMakeLists.txt. These follow -# test_main, test_spice and test_cdf in Linux.mak and Windows.mak. Tests run +# test_main, test_spice and test_cdf in Linux.mak and windows.mak. Tests run # from the top of the source tree and write their output to the build # directory. Labels: main, spice, cdf, e.g. ctest -L cdf # diff --git a/buildfiles/emscripten.mak b/buildfiles/emscripten.mak new file mode 100644 index 0000000..ffb9954 --- /dev/null +++ b/buildfiles/emscripten.mak @@ -0,0 +1,437 @@ +############################################################################## +# WebAssembly cross build via emscripten +# +# Its own entry point, like windows.mak: the top level Makefile never reads +# this file. Run from the top of the source tree: +# +# make -f buildfiles/emscripten.mak Same as build +# make -f buildfiles/emscripten.mak build Static library +# make -f buildfiles/emscripten.mak build_test Adds the test programs +# make -f buildfiles/emscripten.mak test Build, then run the tests +# make -f buildfiles/emscripten.mak PREFIX=/opt/das dev_install +# make -f buildfiles/emscripten.mak clean Removes build.wasm32 +# make -f buildfiles/emscripten.mak distclean Also removes deps.wasm32 +# +# Builds the library archive and, for 'build_test' and 'test', a subset of +# the test programs as node.js scripts; 'build_test' links them without +# running them. No utilities, no shared object, no program install. +# The archive is meant to be linked into a wasm module by a downstream +# project that owns the JavaScript side (das2js). +# +# Prerequisites: +# +# emsdk activated in the calling shell so emcc and emar are on PATH. The +# bundled node is found through EMSDK_NODE, which emsdk_env.sh exports. +# +# Network access on the first build, see below. +# +# Dependencies. Emscripten ships a zlib port (--use-port=zlib) but no expat +# or FFTW, so both are built here for wasm32, the way CDF and CSPICE are for +# native builds: when a pair below is unset, the release tarball is fetched +# from its *_URL, checked against its *_SHA256, and configured, built and +# installed under $(DEPS_DIR). Only distclean removes them. Set a pair to +# use an existing wasm32 build instead, as absolute paths (or paths relative +# to this directory): +# +# EXPAT_INC, EXPAT_LIB directory holding expat.h, and libexpat.a +# FFTW_INC, FFTW_LIB directory holding fftw3.h, and libfftw3.a +# +# What is left out, and why: +# +# http.c credentials.c catalog.c sockets, OpenSSL and the node cache do +# not exist in a browser. fetch() on the JS +# side drives DasNodeWalk instead; node.c, +# source.c and req.c never touch a transport +# cli.c send.c program scaffolding, nothing a library +# consumer needs +# +# Dropping the network sources leaves dangling calls in util.c, io.c and +# buffer.c. DAS_NO_NETWORK compiles those out; das3/defs.h defines it +# whenever __EMSCRIPTEN__ is, so nothing here passes it. +# +# dft.c is in by default. DFT=no (or any value but "yes") leaves it out and +# defines DAS_NO_DFT, which skips FFTW setup in das_init(). FFTW is GPL, so +# a module linked with it is too, see THIRD-PARTY.txt. + +# Every path below is relative to the top of the source tree +ifeq ($(wildcard buildfiles/sources.mak),) +$(error Run make -f buildfiles/emscripten.mak from the top of the das2C source tree) +endif + +# Cleaning doesn't need the compiler +ifeq ($(filter clean distclean,$(MAKECMDGOALS)),) +ifeq ($(shell command -v emcc 2>/dev/null),) +$(error emcc not found on PATH. Activate emsdk first: source /path/to/emsdk/emsdk_env.sh) +endif +endif + +############################################################################## +# Output and install locations + +# One directory per target, as in the native builds. The downloads and +# dependency builds sit beside the build directory rather than in it, so +# clean can remove all of $(BD) and only distclean costs a re-download. +N_ARCH=wasm32 +BD:=build.$(N_ARCH) +DEPS_DIR:=deps.$(N_ARCH) + +# Same defaults as the top level Makefile. Only dev_install uses them, and +# the archive goes under lib/wasm32 so it never lands on a native one. +ifeq ($(PREFIX),) +PREFIX=$(HOME) +endif + +ifeq ($(INST_INC),) +INST_INC=$(PREFIX)/include +endif + +ifeq ($(INST_NAT_LIB),) +INST_NAT_LIB=$(PREFIX)/lib/$(N_ARCH) +endif + +############################################################################## +# Project definitions + +TARG=libdas3 + +include buildfiles/sources.mak + +NET_SRCS:=das3/http.c das3/credentials.c das3/catalog.c +PROG_SRCS:=das3/cli.c das3/send.c +DFT_SRCS:=das3/dft.c + +WASM_SRCS:=$(filter-out $(NET_SRCS) $(PROG_SRCS),$(LIB_SRCS)) +FEATURE_DEFS:= + +DFT?=yes + +# Pulled from the maintainer's site so the downloads count for them. If a +# build fails because fftw.org is down, our department mirror has the same +# file: +# +# make FFTW_URL=https://space.physics.uiowa.edu/spdr/packages/source/$(FFTW_DIST).tar.gz +# +# Upstream publishes only an MD5. The SHA-256 was taken from an fftw.org +# copy that matched it, and holds either source to the release itself. +FFTW_DIST:=fftw-3.3.11 +FFTW_URL?=https://www.fftw.org/$(FFTW_DIST).tar.gz +FFTW_SHA256:=5630c24cdeb33b131612f7eb4b1a9934234754f9f388ff8617458d0be6f239a1 + +# shasum -a 256 on macOS +SHA256SUM?=sha256sum + +BLD_FFTW=0 +ifeq ($(DFT),yes) +ifeq ($(FFTW_INC),) +BLD_FFTW=1 +endif +ifeq ($(FFTW_LIB),) +BLD_FFTW=1 +endif +endif + +ifeq ($(BLD_FFTW),1) +FFTW_INC:=$(DEPS_DIR)/fftw/include +FFTW_LIB:=$(DEPS_DIR)/fftw/lib/libfftw3.a +endif + +ifeq ($(DFT),yes) +DEP_INC:=-I$(FFTW_INC) +else +WASM_SRCS:=$(filter-out $(DFT_SRCS),$(WASM_SRCS)) +FEATURE_DEFS:=-DDAS_NO_DFT +endif + +# Pulled from the maintainer's GitHub releases, as with FFTW above. If that +# is down, our department mirror has the same file: +# +# make EXPAT_URL=https://space.physics.uiowa.edu/spdr/packages/source/$(EXPAT_DIST).tar.gz +# +# The SHA-256 is the one GitHub lists for the release asset +EXPAT_VER:=2.8.5 +EXPAT_DIST:=expat-$(EXPAT_VER) +EXPAT_URL?=https://github.com/libexpat/libexpat/releases/download/R_$(subst .,_,$(EXPAT_VER))/$(EXPAT_DIST).tar.gz +EXPAT_SHA256:=920dde485e15eda0cce8d2310b41d492c534e5e3d89ad407a0b4176dd2ff88fe + +BLD_EXPAT=0 +ifeq ($(EXPAT_INC),) +BLD_EXPAT=1 +endif +ifeq ($(EXPAT_LIB),) +BLD_EXPAT=1 +endif + +ifeq ($(BLD_EXPAT),1) +EXPAT_INC:=$(DEPS_DIR)/expat/include +EXPAT_LIB:=$(DEPS_DIR)/expat/lib/libexpat.a +endif + +DEP_INC:=$(DEP_INC) -I$(EXPAT_INC) + +SRCS:=$(notdir $(WASM_SRCS)) +HDRS:=$(notdir $(LIB_HDRS)) + +# Test programs that need nothing from the excluded sources. Anything that +# opens a URL, reads credentials, or needs SPICE stays out. +TEST_PROGS:=TestValue TestDatum TestGen TestForm TestCplx TestVar \ + TestVarSubset TestDim TestDesc TestUnits TestTT2000 TestArray TestDs TestIter \ + TestRaggedEncode TestFilter TestUri TestV3Read TestFeed TestWalk + +############################################################################## +# Build definitions + +CC=emcc +AR=emar + +# emcc is clang: the gcc-only -Wno-stringop-truncation of Linux.mak would +# itself be an error here +WARNINGS:=-Wall -Werror -Wno-deprecated-declarations + +DEFINES:=-D_XOPEN_SOURCE=600 $(FEATURE_DEFS) + +OPTFLAGS=-O2 -g + +# --use-port=zlib supplies zlib.h at compile time and the library at link +# time. -MMD -MP writes a .d beside each .o for header dependency tracking, +# same scheme as Linux.mak. +CFLAGS=$(WARNINGS) $(OPTFLAGS) $(DEFINES) -std=c99 -I. $(DEP_INC) \ + --use-port=zlib -MMD -MP + +# Test programs run under node with the real filesystem mounted (NODERAWFS) +# so fixtures need no embedding. Flag notes: +# * EXIT_RUNTIME makes main()'s return value the process exit status +# * The 1 MiB stack is native-like; emscripten defaults to 64 KiB. +# * -g keeps DWARF, which makes emcc warn that wasm-opt ran fewer passes +# with -Werror on the same command line that would fail the link. +# +TEST_LFLAGS=--use-port=zlib -sENVIRONMENT=node -sNODERAWFS=1 -sEXIT_RUNTIME=1 \ + -sALLOW_MEMORY_GROWTH=1 -sSTACK_SIZE=1048576 \ + -Wno-limited-postlink-optimizations + +ifeq ($(NODE),) +ifeq ($(EMSDK_NODE),) +NODE:=node +else +NODE:=$(EMSDK_NODE) +endif +endif + +############################################################################## +# Derived definitions + +BUILD_OBJS=$(patsubst %.c,$(BD)/%.o,$(SRCS)) + +BUILD_TEST_PROGS=$(patsubst %,$(BD)/%.js,$(TEST_PROGS)) + +INST_HDRS=$(patsubst %.h,$(DESTDIR)$(INST_INC)/das3/%.h,$(HDRS)) + +# Fixture lists mirror Linux.mak, see the comment there for the exclusions +V3_FIXTURES := \ + $(filter-out examples/ex28_%,$(wildcard examples/*.d3b examples/*.d3t)) \ + $(filter-out test/streams/notimp_% test/streams/reject_%,$(wildcard test/streams/*.d3b test/streams/*.d3t)) + +V2_FIXTURES := $(wildcard test/streams/*.d2s test/streams/*.d2t examples/*.d2s examples/*.d2t) + +############################################################################## +# Pattern rules + +$(BD)/%.o:das3/%.c | $(BD) + $(CC) -c $(CFLAGS) -o $@ $< + +# emcc picks the output kind from the suffix: a .js loader with the .wasm +# beside it. The dependency libraries are prerequisites so that a rebuilt +# or caller-supplied one relinks these. FFTW_LIB is empty with DFT off. +$(BD)/%.js:test/%.c $(BD)/$(TARG).a $(EXPAT_LIB) $(FFTW_LIB) | $(BD) + $(CC) $(CFLAGS) $< $(BD)/$(TARG).a $(EXPAT_LIB) $(FFTW_LIB) $(TEST_LFLAGS) -o $@ + +$(DESTDIR)$(INST_NAT_LIB)/%.a:$(BD)/%.a + install -D -m 664 $< $@ + +$(DESTDIR)$(INST_INC)/das3/%.h:das3/%.h + install -D -m 664 $< $@ + +# No .SECONDARY here, unlike Linux.mak: nothing is an intermediate. Objects +# are explicit prereqs of the archive and the test .js files link straight from +# test/*.c. +.PHONY: build build_test test install dev_install clean distclean + +############################################################################## +# Explicit rules + +build: $(BD)/$(TARG).a + +$(BD): + @if [ ! -e "$(BD)" ]; then echo mkdir $(BD); mkdir $(BD); chmod g+w $(BD); fi + +$(DEPS_DIR): + @if [ ! -e "$(DEPS_DIR)" ]; then echo mkdir $(DEPS_DIR); mkdir $(DEPS_DIR); chmod g+w $(DEPS_DIR); fi + +# expat, when not supplied ################################################### + +ifeq ($(BLD_EXPAT),1) +# The sources that include expat.h, which the install below puts in place +$(BD)/dataset_hdr3.o $(BD)/oob.o $(BD)/packet.o $(BD)/stream.o: | $(EXPAT_LIB) +endif + +$(DEPS_DIR)/$(EXPAT_DIST).tar.gz: | $(DEPS_DIR) + curl -f -L $(EXPAT_URL) > $@.part + echo "$(EXPAT_SHA256) $@.part" | $(SHA256SUM) -c - + mv $@.part $@ + +# Order-only, see the FFTW rule below for why +$(DEPS_DIR)/$(EXPAT_DIST)/configure: | $(DEPS_DIR)/$(EXPAT_DIST).tar.gz + cd $(DEPS_DIR) && tar -xzf $(EXPAT_DIST).tar.gz + +# Static library only, no xmlwf, examples, tests or man pages. Parser +# features stay at expat's defaults (DTD, general entities, namespaces, 1024 +# context bytes), the same as the system expat native builds link, so both +# parse alike. The hash salt comes from getentropy() alone, which emscripten +# implements over crypto.getRandomValues (browser) or crypto.randomFillSync +# (node). Every other entropy source is refused: the browser module has no +# /dev/urandom, and a probe that guessed wrong would fail at runtime, not +# here. +# +# emconfigure is needed, emmake is not. configure records the full paths +# of emcc, emar and emranlib in the Makefiles it writes, so a plain make +# already builds for wasm. emmake would also cost make -j: it is a Python +# wrapper, Python closes inherited file descriptors when it starts a +# program, and those carry make's job slots, so the sub-make warns +# "jobserver unavailable" and runs one job at a time. The FFTW rule below +# is where that hurts. +$(EXPAT_LIB): $(DEPS_DIR)/$(EXPAT_DIST)/configure + cd $(DEPS_DIR)/$(EXPAT_DIST) && emconfigure ./configure \ + --prefix=$(CURDIR)/$(DEPS_DIR)/expat --disable-shared --enable-static \ + --without-xmlwf --without-examples --without-tests --without-docbook \ + --with-getentropy --without-getrandom --without-sys-getrandom \ + --without-arc4random --without-arc4random-buf --without-dev-urandom + cd $(DEPS_DIR)/$(EXPAT_DIST) && $(MAKE) install + +# FFTW, when not supplied #################################################### + +ifeq ($(BLD_FFTW),1) +# Only dft.c includes fftw3.h, which the install below puts in place +$(BD)/dft.o: | $(FFTW_LIB) +endif + +# Kept only once the hash matches, so a short or swapped download never +# looks finished to make +$(DEPS_DIR)/$(FFTW_DIST).tar.gz: | $(DEPS_DIR) + curl -f -L $(FFTW_URL) > $@.part + echo "$(FFTW_SHA256) $@.part" | $(SHA256SUM) -c - + mv $@.part $@ + +# Extracted files keep the release's time stamps, which autotools reads: +# stamped all alike (tar -m), Makefile.am looks no older than Makefile.in +# and make tries to rerun automake. So configure is older than the tarball, +# and the prerequisite is order-only to keep that from forcing a re-extract. +$(DEPS_DIR)/$(FFTW_DIST)/configure: | $(DEPS_DIR)/$(FFTW_DIST).tar.gz + cd $(DEPS_DIR) && tar -xzf $(FFTW_DIST).tar.gz + +# Static, double precision only: dft.c uses the fftw_ API alone. No threads +# (this wasm build is single threaded), no Fortran, no docs. The install +# prefix is where FFTW_INC and FFTW_LIB point. CFLAGS is what configure +# picks on its own minus -mtune=native, which clang ignores for wasm with a +# warning per file, well over a thousand of them. +# +# SIMD: FFTW's generic-simd128 codelets are written with compiler vector +# extensions, and -msimd128 has clang lower them to wasm SIMD, which every +# current browser and node runs (Firefox since 89). If a build or a runtime +# fails over it, drop both --enable-generic-simd128 and -msimd128, then make +# distclean. +# +# ac_cv_func_clock_gettime=no is not optional. With clock_gettime found, +# kernel/cycle.h takes clock_gettime(CLOCK_SGI_CYCLE) as its tick counter +# because musl defines that constant, but no wasm runtime has the clock. +# The ticks never advance and FFTW_MEASURE planning loops forever. Without +# it FFTW has no tick counter and plans by estimate, its own fallback. +# Nothing else in FFTW uses clock_gettime. +$(FFTW_LIB): $(DEPS_DIR)/$(FFTW_DIST)/configure + cd $(DEPS_DIR)/$(FFTW_DIST) && emconfigure ./configure \ + --prefix=$(CURDIR)/$(DEPS_DIR)/fftw --disable-shared --enable-static \ + --disable-fortran --disable-threads --disable-doc \ + --enable-generic-simd128 ac_cv_func_clock_gettime=no \ + CFLAGS="-O3 -fomit-frame-pointer -fstrict-aliasing -msimd128" + cd $(DEPS_DIR)/$(FFTW_DIST) && $(MAKE) install + +# Records which objects make up the library. DFT= changes that set without +# making any object newer than the archive, so make alone would keep a stale +# one. The file is rewritten only when the set differs. +$(BD)/lib_members: FORCE | $(BD) + @echo '$(BUILD_OBJS)' | cmp -s - $@ || echo '$(BUILD_OBJS)' > $@ + +FORCE: +.PHONY: FORCE + +# The member set depends on DFT, so start the archive fresh rather than let +# ar keep stale objects from an earlier configuration +$(BD)/$(TARG).a:$(BUILD_OBJS) $(BD)/lib_members | $(BD) + rm -f $@ + $(AR) rc $@ $(BUILD_OBJS) + +# Link only, run nothing: lets a parent build make the test programs in its +# default target and run them later +build_test: $(BD)/$(TARG).a $(BUILD_TEST_PROGS) + +test: build_test + @echo "INFO: wasm test programs run under $(NODE)" + @echo "INFO: Running unit test for the value layer, $(BD)/TestValue..." + @$(NODE) $(BD)/TestValue.js + @echo "INFO: Running unit test for the datum layer, $(BD)/TestDatum..." + @$(NODE) $(BD)/TestDatum.js + @echo "INFO: Running unit test for the DasGen value sources, $(BD)/TestGen..." + @$(NODE) $(BD)/TestGen.js + @echo "INFO: Running unit test for the DasForm formalisms, $(BD)/TestForm..." + @$(NODE) $(BD)/TestForm.js + @echo "INFO: Running unit test for complex arithmetic, $(BD)/TestCplx..." + @$(NODE) $(BD)/TestCplx.js + @echo "INFO: Running unit test for the DasVar layer, $(BD)/TestVar..." + @$(NODE) $(BD)/TestVar.js + @echo "INFO: Running unit test for DasVar bulk reads, $(BD)/TestVarSubset..." + @$(NODE) $(BD)/TestVarSubset.js + @echo "INFO: Running unit test for the DasDim container, $(BD)/TestDim..." + @$(NODE) $(BD)/TestDim.js + @echo "INFO: Running unit test for descriptor property storage, $(BD)/TestDesc..." + @$(NODE) $(BD)/TestDesc.js + @echo "INFO: Running unit test to test units, $(BD)/TestUnits..." + @$(NODE) $(BD)/TestUnits.js + @echo "INFO: Running unit test for TT2000 leap seconds, $(BD)/TestTT2000..." + @$(NODE) $(BD)/TestTT2000.js + @echo "INFO: Running unit test for dynamic arrays, $(BD)/TestArray..." + @$(NODE) $(BD)/TestArray.js + @echo "INFO: Running unit test for dataset shape/length merge, $(BD)/TestDs..." + @$(NODE) $(BD)/TestDs.js + @echo "INFO: Running unit test for stream parsing over all fixtures, $(BD)/TestV3Read..." + $(NODE) $(BD)/TestV3Read.js test/streams/tag_test.dNt $(V2_FIXTURES) $(V3_FIXTURES) + @echo "INFO: Running unit test for push parsing vs. pull parsing, $(BD)/TestFeed..." + $(NODE) $(BD)/TestFeed.js test/streams/tag_test.dNt $(V2_FIXTURES) $(V3_FIXTURES) + @echo "INFO: Running unit test for ragged and unique iteration, $(BD)/TestIter..." + $(NODE) $(BD)/TestIter.js + @echo "INFO: Running unit test for ragged binary re-encode, $(BD)/TestRaggedEncode..." + $(NODE) $(BD)/TestRaggedEncode.js examples/ex30_cassini_ragged_notlast.d3b \ + examples/ex31_efi_ragged_vec.d3b examples/ex32_marsis_2d_ragged.d3b \ + examples/ex34_ragged_fixstr.d3b examples/ex38_wbr_wfrm_tags.d3b \ + examples/ex39_sandwich.d3b + @echo "INFO: Running unit test for filter dataset ops (copy/swap), $(BD)/TestFilter..." + $(NODE) $(BD)/TestFilter.js + @echo "INFO: Running unit test for URI templates, $(BD)/TestUri..." + $(NODE) $(BD)/TestUri.js $(BD)/uri_tplt + @echo "INFO: Running unit test for catalog walks over local nodes, $(BD)/TestWalk..." + @$(NODE) $(BD)/TestWalk.js + @echo "INFO: ===============================================" + @echo "INFO: All wasm test programs completed without errors" + @echo "INFO: ===============================================" + +# Only the archive and headers: there are no programs to install +dev_install: $(DESTDIR)$(INST_NAT_LIB)/$(TARG).a $(INST_HDRS) +install: dev_install + +# All of $(BD) is ours; $(DEPS_DIR) holds the expat and FFTW downloads and +# builds +clean: + if [ -d "$(BD)" ] ; then rm -r $(BD) ; fi + +distclean: clean + if [ -d "$(DEPS_DIR)" ] ; then rm -r $(DEPS_DIR) ; fi + +# Header dependencies written by -MMD above +-include $(wildcard $(BD)/*.d) diff --git a/buildfiles/sources.mak b/buildfiles/sources.mak index f9a3178..6a6803c 100644 --- a/buildfiles/sources.mak +++ b/buildfiles/sources.mak @@ -1,4 +1,4 @@ -# Source lists shared by Linux.mak and Windows.mak +# Source lists shared by Linux.mak and windows.mak # # This file is read by both GNU make and nmake, so stick to the subset of # syntax they share: plain '=' assignments, '#' comments, and backslash line diff --git a/buildfiles/Windows.mak b/buildfiles/windows.mak similarity index 98% rename from buildfiles/Windows.mak rename to buildfiles/windows.mak index da31a9a..0f042b1 100644 --- a/buildfiles/Windows.mak +++ b/buildfiles/windows.mak @@ -3,12 +3,12 @@ # Run from the top of the source tree in a 64-bit build environment, i.e. # after vcvars64.bat, or in an "x64 Native Tools Command Prompt": # -# nmake /f buildfiles\Windows.mak DLL, static library and programs -# nmake /f buildfiles\Windows.mak test Build, then run the tests -# nmake /f buildfiles\Windows.mak dll das3.dll and import lib das3.lib -# nmake /f buildfiles\Windows.mak PREFIX=C:\opt install Programs, DLL, schemas -# nmake /f buildfiles\Windows.mak PREFIX=C:\opt dev_install Adds static lib, headers -# nmake /f buildfiles\Windows.mak PREFIX=C:\opt dll_install DLL, import lib, headers +# nmake /f buildfiles\windows.mak DLL, static library and programs +# nmake /f buildfiles\windows.mak test Build, then run the tests +# nmake /f buildfiles\windows.mak dll das3.dll and import lib das3.lib +# nmake /f buildfiles\windows.mak PREFIX=C:\opt install Programs, DLL, schemas +# nmake /f buildfiles\windows.mak PREFIX=C:\opt dev_install Adds static lib, headers +# nmake /f buildfiles\windows.mak PREFIX=C:\opt dll_install DLL, import lib, headers # # Add SPICE=yes and/or CDF=yes to any of the above to build in the SPICE and # CDF programs. @@ -43,7 +43,7 @@ # are installed. Thus the public targets fetch dependencies first and then # run nmake again for the actual build. -THIS=buildfiles\Windows.mak +THIS=buildfiles\windows.mak MAKE=nmake /nologo CC=cl.exe /nologo diff --git a/das3/buffer.c b/das3/buffer.c index 8c578fe..5d528d6 100644 --- a/das3/buffer.c +++ b/das3/buffer.c @@ -24,7 +24,9 @@ #include /* Sockets and OpenSSL serve only the two writeFrom* helpers at the bottom. - A DAS_NO_NETWORK build (wasm, see buildfiles/Emscripten.mak) drops both. */ + A DAS_NO_NETWORK build (wasm, see defs.h) drops both. defs.h comes first + because it decides DAS_NO_NETWORK. */ +#include "defs.h" #ifndef DAS_NO_NETWORK #ifndef _WIN32 #include diff --git a/das3/defs.h b/das3/defs.h index bfc5a69..58d6d31 100644 --- a/das3/defs.h +++ b/das3/defs.h @@ -99,10 +99,12 @@ #endif /* __sun */ #endif /* __linux */ -/* Setup the DLL macros for windows. GCC and clang builds that compile with - -fvisibility=hidden export only DAS_API names from a shared library, the - same set a Windows DLL exports. Emscripten is left alone, its exports come - from the link line. */ +/* disable code that needs sockets for wasm targets */ +#ifdef __EMSCRIPTEN__ +#define DAS_NO_NETWORK +#endif + +/* Export only DAS_API names from shared libraies on all platforms */ #if defined(_WIN32) && defined(DAS_USE_DLL) # ifdef BUILDING_DLL # define DAS_API __declspec(dllexport) @@ -122,9 +124,11 @@ #endif /* Ask the compiler to check printf-style calls the way it checks printf - itself. FMT_IDX and ARG_IDX are 1-based parameter positions of the format - string and the first variadic argument. Not part of any C standard; - expands to nothing where the compiler lacks it. */ + itself if the compiler provides this feature, ignored otherwise. + + Not part of any C standard: FMT_IDX and ARG_IDX are 1-based parameter + positions of the format string and the first variadic argument. +*/ #if defined(__GNUC__) || defined(__clang__) #define _das_fmt_check(FMT_IDX, ARG_IDX) \ __attribute__((__format__(__printf__, FMT_IDX, ARG_IDX))) @@ -164,10 +168,9 @@ extern "C" { * @{ */ -/* Stream identification. These live here rather than in stream.h so that a - framing-only tool (one that just needs to know how the bytes are chunked, - say a WebSocket reader) can know them without pulling in the header - parser. The STREAM_ prefix predates the move and stays for compatibility. */ +/* Stream identification. Provided here rather than in stream.h so that a + framing-only tool (packet tag reader) dosen't need to pull in the full + data model. */ /** In-memory object model a reader builds, or a writer emits. * @@ -183,16 +186,14 @@ typedef enum das_stream_model { /** The version="" attribute of a stream header: schema generation. * * For packetized output this also fixes the packet tag style. Values are - * major*10 + minor so that ordering comparisons work. The 1.x members name - * das1 formats that have no in-band header; a DasStream never reports them, - * they exist so the one list of stream versions lives in one place. + * major*10 + minor so that ordering comparisons work. */ typedef enum das_stream_ver { STREAM_VER_UNK = 0, /* No header seen yet, or a version this library doesn't know */ - STREAM_VER_1_0 = 10, /* das1, fixed records, no header in band */ - STREAM_VER_1_1 = 11, /* tagged das1 */ - STREAM_VER_2_2 = 22, - STREAM_VER_3_0 = 30 + STREAM_VER_1_0 = 10, /* das1, original big-endian fixed records no hdrs*/ + STREAM_VER_1_1 = 11, /* tagged das1, an interum step */ + STREAM_VER_2_2 = 22, /* das2 the decades long workhorse */ + STREAM_VER_3_0 = 30 /* das3 the new-ish kid on the block */ } das_stream_ver; /** The type="" attribute of a stream header: container and serialization. diff --git a/das3/dft.c b/das3/dft.c index fd542d2..0c4f402 100644 --- a/das3/dft.c +++ b/das3/dft.c @@ -43,6 +43,14 @@ #define M_PI 3.1415926535897931 #endif +/* No wisdom on wasm.* Wisdom only fits the FFTW build and CPU it was + measured on, so a native file is no use to a wasm libfftw3. There's no + cycle counter to measure with either, and the browser module has no + filesystem to read one from. + + *some may beg to differ +*/ +#ifndef __EMSCRIPTEN__ #ifndef WISDOM_FILE #error WISDOM_FILE should be defined to the system fftw wisdom location #endif @@ -50,12 +58,23 @@ #define _QDEF(x) #x #define QDEF(x) _QDEF(x) #define DEF_WISDOM QDEF(WISDOM_FILE) +#endif + +/* Nothing to measure with on wasm either, and worse, FFTW picks + clock_gettime(CLOCK_SGI_CYCLE) for its tick counter which no + runtime supports and the ticks never advance. */ +#ifdef __EMSCRIPTEN__ +#define DAS_FFTW_PLANNER FFTW_ESTIMATE +#else +#define DAS_FFTW_PLANNER FFTW_MEASURE +#endif static int g_nExecCount = 0; static pthread_mutex_t g_mtxExecCount = PTHREAD_MUTEX_INITIALIZER; static pthread_cond_t g_cndExecCountDec = PTHREAD_COND_INITIALIZER; bool das_dft_init(const char* sProgName){ +#ifndef __EMSCRIPTEN__ /* import the system wide FFTW wisdom */ bool bHaveWisdomFile = false; FILE* pWis = NULL; @@ -88,7 +107,8 @@ bool das_dft_init(const char* sProgName){ DEF_WISDOM, sProgName ); } - +#endif + pthread_mutexattr_t attr; pthread_mutexattr_init(&attr); #ifndef NDEBUG @@ -160,7 +180,7 @@ DftPlan* new_DftPlan(size_t uLen, bool bForward) pThis->vpPlan = fftw_plan_dft_1d( uLen, (fftw_complex*)pThis->vpIn, (fftw_complex*)pThis->vpOut, - nSign, FFTW_MEASURE + nSign, DAS_FFTW_PLANNER ); pThis->uLen = uLen; diff --git a/das3/io.c b/das3/io.c index ee40005..608e410 100644 --- a/das3/io.c +++ b/das3/io.c @@ -41,9 +41,9 @@ #define gettimeofday win_gettimeofday #endif -/* DAS_NO_NETWORK (the wasm build, see buildfiles/Emscripten.mak) has no - sockets and no OpenSSL. The socket and SSL stream modes compile out - entirely rather than failing at runtime */ +/* DAS_NO_NETWORK (the wasm build, see defs.h) has no sockets and no + OpenSSL. Always include defs first, it sets the feature list. */ +#include "defs.h" #ifndef DAS_NO_NETWORK #include #endif diff --git a/das3/util.c b/das3/util.c index 62bc291..f50195b 100644 --- a/das3/util.c +++ b/das3/util.c @@ -161,15 +161,15 @@ void das_init( das_error(DASERR_INIT, "(%s) Failed units initialization", sProgName); exit(DASERR_INIT); } - /* Builds without FFTW or sockets (DAS_NO_DFT, DAS_NO_NETWORK, see - buildfiles/Emscripten.mak) have no dft.c or http.c to initialize */ +/* FFTW support enabled/disable see defs.h) */ #ifndef DAS_NO_DFT if( ! das_dft_init(sProgName) ){ das_error(DASERR_INIT, "(%s) Failed DFT initialization", sProgName); exit(DASERR_INIT); } #endif - + +/* network support enabled/disable see defs.h) */ #ifndef DAS_NO_NETWORK if( ! das_http_init(sProgName)){ das_error(DASERR_INIT, "(%s) Failed HTTP initialization", sProgName); diff --git a/notes/build_for_emscripten.md b/notes/build_for_emscripten.md index e80291c..66d18c8 100644 --- a/notes/build_for_emscripten.md +++ b/notes/build_for_emscripten.md @@ -1,10 +1,12 @@ # Building das2C for WebAssembly -`make EMSCRIPTEN=yes` cross-compiles `libdas3.a` to wasm32 with the -[emscripten](https://emscripten.org) toolchain. Only the library and its -unit tests are built; the command line utilities are not, since a wasm -module runs inside some other program (a browser page, a node script) that -supplies its own I/O. The first consumer is +`make -f buildfiles/emscripten.mak` cross-compiles `libdas3.a` to wasm32 +with the [emscripten](https://emscripten.org) toolchain. Like +`buildfiles/windows.mak` it is its own entry point; the top level +`Makefile` never reads it. Only the library and its unit tests are built; +the command line utilities are not, since a wasm module runs inside some +other program (a browser page, a node script) that supplies its own I/O. +The first consumer is [das2js](https://github.com/das-developers/das2js), which wraps the archive in a flat C glue layer and an ES module. Nothing in das2C knows about that layer: no export markup, no emscripten headers beyond the byte-order probe @@ -21,74 +23,96 @@ $ cd emsdk && ./emsdk install latest && ./emsdk activate latest $ source ./emsdk_env.sh ``` -The emsdk tools are Python programs needing 3.8 or newer. On a host whose +The emsdk tools are Python programs needing 3.10 or newer. On a host whose default `python3` is older, point `EMSDK_PYTHON` at a newer interpreter; note that `emsdk_env.sh` clears the variable, so export it after sourcing as well as before. The script does not put its node on `PATH` either; the -test target reads `EMSDK_NODE`, which it does export. Built with emscripten -6.0 (clang 23) in 2026-10; anything recent should do. +test target reads `EMSDK_NODE`, which it does export. Last built with +emscripten 6.0.13 in 2026-10, the version `.github/workflows/emscripten.yml` +pins; anything recent should do. -## Expat +## Dependencies -Emscripten ships a zlib port, used via `--use-port=zlib`, but no expat port, -so a libexpat compiled for wasm32 has to come from somewhere. Pass its -header directory as `EXPAT_INC` for the build, and the archive as -`EXPAT_LIB` to link the tests. This is the same shape as `CSPICE_INC` and -`CDF_INC` on the native builds: das2C never learns where the dependency -lives. +Emscripten ships a zlib port, used via `--use-port=zlib`, but no expat or +FFTW. The first build downloads both release tarballs, checks each against +a pinned SHA-256, and configures, builds and installs them for wasm32 under +`deps.wasm32/`, the way the native builds fetch CDF and CSPICE. So the first +build needs network access; later ones reuse `deps.wasm32/` until a +`distclean`. If a download site is down, the `EXPAT_URL` and `FFTW_URL` +comments in `buildfiles/emscripten.mak` give a mirror. -Two ways to get one. libexpat's autotools build works under `emconfigure` -and `emmake`. Or, since the parser is only three source files plus one -entropy source, compile them directly against a hand-written -`expat_config.h`; das2js does that from a pinned libexpat submodule with a -20-line config header, see its `buildfiles/expat_config.h` and `Makefile`. +To use copies already built for wasm32 instead, set both variables of a +pair: + +| variables | point at | +|--------------------------|------------------------------------------------| +| `EXPAT_INC`, `EXPAT_LIB` | the directory holding `expat.h`, `libexpat.a` | +| `FFTW_INC`, `FFTW_LIB` | the directory holding `fftw3.h`, `libfftw3.a` | + +FFTW is GPL, see `THIRD-PARTY.txt`. It is in `libdas3.a` through `dft.c`, +but a linker only carries it into a program that calls a transform. ## Build and test ```bash -$ make EMSCRIPTEN=yes EXPAT_INC=/path/to/expat/lib build -$ make EMSCRIPTEN=yes EXPAT_INC=/path/to/expat/lib EXPAT_LIB=/path/to/libexpat.a test +$ make -f buildfiles/emscripten.mak build +$ make -f buildfiles/emscripten.mak test ``` -Output lands in `build.wasm32/`. The archive is `libdas3.a`, 43 objects -in 2026-10. Each test program is a `Test*.js` loader plus `Test*.wasm`, -run under node with the real filesystem mounted (`-sNODERAWFS`) so the -fixtures in `examples/` and `test/` are read in place. `make test` is slow, -several minutes, almost all of it `TestFeed` re-feeding every fixture one -byte at a time; that is call overhead, not parsing speed. +Run these from the top of the source tree. Output lands in +`build.wasm32/`. The archive is `libdas3.a`, 47 objects in 2026-10. Each +test program is a `Test*.js` loader plus `Test*.wasm`, run under node with +the real filesystem mounted (`-sNODERAWFS`) so the fixtures in `examples/` +and `test/` are read in place. `make test` is slow, several minutes, almost +all of it `TestFeed` re-feeding every fixture one byte at a time; that is +call overhead, not parsing speed. `build_test` links the test programs +without running them. -`make EMSCRIPTEN=yes clean` removes `build.wasm32/` and needs neither emcc -nor `EXPAT_INC`. +`clean` removes `build.wasm32/`; `distclean` also removes `deps.wasm32/`, +the expat and FFTW downloads and builds. Neither needs emcc. ## What is left out, and how to get it back -`buildfiles/Emscripten.mak` filters `buildfiles/sources.mak` rather than +`buildfiles/emscripten.mak` filters `buildfiles/sources.mak` rather than keeping a second source list. Its header comment is the authoritative version of this table. -| left out | why | switch | -|--------------------------------------|---------------------------------------|----------------------| -| `http.c`, `credentials.c`, `node.c` | sockets and OpenSSL; browsers do HTTP | none, see below | -| `dft.c` | needs FFTW | `DFT=yes FFTW_INC=` | -| `cli.c`, `send.c` | program scaffolding for utilities | none | +| left out | why | switch | +|-----------------------------------------|---------------------------------------|----------| +| `http.c`, `credentials.c`, `catalog.c` | sockets and OpenSSL; browsers do HTTP | none | +| `cli.c`, `send.c` | program scaffolding for utilities | none | +| `dft.c` | in by default | `DFT=no` | `spice.h` has no library code behind it, so there's nothing to leave out for SPICE. A program that includes it needs CSPICE, which isn't built for wasm. -Two feature macros, `DAS_NO_NETWORK` and `DAS_NO_DFT`, are defined by -`Emscripten.mak` and nowhere else. They compile the socket and SSL stream -modes out of `io.c` and `buffer.c` and the FFTW and HTTP setup out of -`util.c`. No public header changes with them; a program compiled against -the installed headers sees the same API, and the socket constructors are -simply absent from the archive. +Two feature macros compile code out. `DAS_NO_NETWORK` is defined by +`das3/defs.h` whenever `__EMSCRIPTEN__` is, since no wasm build has +sockets; it removes the socket and SSL stream modes from `io.c` and +`buffer.c` and the HTTP setup from `util.c`. `DAS_NO_DFT` is defined by +`DFT=no` and removes the FFTW setup from `util.c`. No public header changes +with either; a program compiled against the installed headers sees the same +API, and the left out functions are simply absent from the archive. Network access from wasm is the embedding runtime's job (fetch in a browser, -http in node), feeding bytes to `DasIO_feed`. The catalog walk in `node.c` -is likewise a JSON-and-HTTP exercise better done in the host language. +http in node), feeding bytes to `DasIO_feed`. The catalog walk stays: +`node.c`, `source.c` and `req.c` never touch a transport, so the host side +fetches each node and `DasNodeWalk` does the rest. + +## FFTW on wasm + +FFTW is built with its generic SIMD128 codelets, which clang lowers to wasm +SIMD; every current browser and node runs it. If a build or a runtime +fails over it, the comment above the FFTW rule in `emscripten.mak` says what +to drop. -FFTW is expected back: spectrograms computed in C are a planned strength. -`DFT=yes` with `FFTW_INC` pointing at an FFTW built for wasm32 re-enables -`dft.c` today; nobody has built that FFTW yet. +Plans are made with `FFTW_ESTIMATE`, not the `FFTW_MEASURE` native builds +use, and no wisdom file is read. There is nothing to time with on wasm: +the tick counter FFTW's configure finds there never advances, so measured +planning would never return. `emscripten.mak` configures FFTW without it +and `dft.c` asks for estimates regardless, which covers an FFTW supplied +through `FFTW_LIB`. Wisdom is specific to the machine and build that made +it, so a native wisdom file would be of no use here anyway. ## Platform notes @@ -98,7 +122,7 @@ is undefined, so this is the first platform in years to exercise das2C's libc's text-to-double conversion several times slower than on x86_64 (see das2js `notes/performance.md`); binary streams are unaffected. The libc is musl, not glibc, and the compiler is clang with `-Werror`, so gcc-only -warning flags cannot go in `Emscripten.mak`. Four long-standing bugs +warning flags cannot go in `emscripten.mak`. Four long-standing bugs turned up on the first build, none of them wasm-specific: a `false` returned from a pointer function, float to int64 range checks that let a value exactly on the limit overflow, an uninitialized leap-second table @@ -109,8 +133,9 @@ the next new platform arrives. ## Linking a program of your own The archive is ordinary wasm32 object code. Link it with emcc like any -static library, together with your expat archive and `--use-port=zlib`. -The test programs use +static library, followed by the dependency archives, +`deps.wasm32/fftw/lib/libfftw3.a` and `deps.wasm32/expat/lib/libexpat.a` +(or your own), and `--use-port=zlib`. The test programs use ``` -sENVIRONMENT=node -sNODERAWFS=1 -sEXIT_RUNTIME=1 diff --git a/notes/build_for_macos.md b/notes/build_for_macos.md index 20a845f..d1230ad 100644 --- a/notes/build_for_macos.md +++ b/notes/build_for_macos.md @@ -59,10 +59,10 @@ existing copies of the toolkits, set `CSPICE_INC`, `CSPICE_LIB`, `CDF_INC` and `CDF_LIB`, otherwise: * CSPICE: The NAIF toolkit for this machine's architecture is downloaded to - `build./cspice`. + `deps./cspice`. * CDF: The SPDF source distribution is downloaded and compiled under - `build./cdf39_1-dist`. While it builds, the CDF makefile prints a few + `deps./cdf39_1-dist`. While it builds, the CDF makefile prints a few `xcodebuild requires Xcode` errors and `ld: warning: search path` messages as it probes for a full Xcode install. These are harmless. diff --git a/notes/build_for_windows.md b/notes/build_for_windows.md index 1323448..6ce6b7f 100644 --- a/notes/build_for_windows.md +++ b/notes/build_for_windows.md @@ -137,8 +137,8 @@ Then get the source, build and test the software: ```bat git clone https://github.com/das-developers/das2C.git cd das2C -nmake /f buildfiles\Windows.mak -nmake /f buildfiles\Windows.mak test +nmake /f buildfiles\windows.mak +nmake /f buildfiles\windows.mak test ``` This builds `das3.dll` with its import library `das3.lib`, the static library @@ -151,9 +151,9 @@ install the static library and headers needed to build against das2C. Use `/n` to see what's going to be installed first. ```bat -nmake /f buildfiles\Windows.mak PREFIX=C:\opt /n install -nmake /f buildfiles\Windows.mak PREFIX=C:\opt install -nmake /f buildfiles\Windows.mak PREFIX=C:\opt dev_install +nmake /f buildfiles\windows.mak PREFIX=C:\opt /n install +nmake /f buildfiles\windows.mak PREFIX=C:\opt install +nmake /f buildfiles\windows.mak PREFIX=C:\opt dev_install ``` The install also copies `vcruntime140.dll` from your Visual Studio's @@ -165,14 +165,14 @@ To build against the DLL instead of the static library, `dll_install` adds the import library and the headers: ```bat -nmake /f buildfiles\Windows.mak PREFIX=C:\opt dll_install +nmake /f buildfiles\windows.mak PREFIX=C:\opt dll_install ``` To use dependency libraries you already have instead of the vcpkg ones (conda does this), set both of the following and `VCPKG_ROOT` isn't needed: ```bat -set LIBRARY_INC= :: directory containing expat.h, fftw3.h, openssl\, etc. +set LIBRARY_INC= :: directory containing expat.h, fftw3.h, openssl, etc. set LIBRARY_LIB= :: directory containing the matching .lib files ``` @@ -185,9 +185,9 @@ The source file lists live in `buildfiles\sources.mak`, which is shared with Add `SPICE=yes` and/or `CDF=yes` to any of the nmake commands above: ```bat -nmake /f buildfiles\Windows.mak SPICE=yes CDF=yes -nmake /f buildfiles\Windows.mak SPICE=yes CDF=yes test -nmake /f buildfiles\Windows.mak SPICE=yes CDF=yes PREFIX=C:\opt install +nmake /f buildfiles\windows.mak SPICE=yes CDF=yes +nmake /f buildfiles\windows.mak SPICE=yes CDF=yes test +nmake /f buildfiles\windows.mak SPICE=yes CDF=yes PREFIX=C:\opt install ``` NAIF does not allow CSPICE to be hosted in a public git repository, and the