Feature/fetch

.gitignore

@@ -12,5 +12,5 @@ test/test_nan
 tmp/*
 CLAUDE.md
 .claude
-co_notes
+c*_notes
 

README.md

@@ -246,6 +246,13 @@ to stdout after all input is complete.  All the rest:
 
 process data packet by packet and output data as soon as it's ready.
 
+The exception at the front of a pipeline is **das_fetch**.  It takes no
+stream input.  Give it a das catalog path and it walks the federated catalog,
+works out the server URL and query string from the data source definition,
+and streams the response body to standard output, whatever format that is.
+Run `das_fetch node uiowa` to start browsing, or `das_fetch -h` for the
+query key rules.
+
 ## Reporting bugs
 Please use the issue tracker for the [das2C](https://github.com/das-developers/das2C/issues) 
 github.com project to report any problems with the library.  If you've fixed a bug, 

buildfiles/Darwin.mak

@@ -217,7 +217,7 @@ $(DESTDIR)$(INST_NAT_BIN)/%:$(BD)/%
 # archive.  Listing those too would let a missing one, say the object for a
 # newly added source, go unbuilt while the archive looks up to date.
 .SECONDARY: $(UTIL_OBJS)
-.PHONY: test build
+.PHONY: test test_more build
 
 ## Explicit Rules  ###########################################################
 
@@ -374,6 +374,14 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS)
 	$(BD)/TestUri $(BD)/uri_tplt
 	@echo "INFO: Running unit test for credentials manager, $(BD)/TestCredMngr..."
 	@$(BD)/TestCredMngr $(BD)
+	@echo "INFO: Running unit test for catalog walks over local nodes, $(BD)/TestWalk..."
+	@$(BD)/TestWalk
+	@echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..."
+	@$(BD)/TestCat
+	@echo "INFO: Running unit test for source descriptions and requisitions, $(BD)/TestSource..."
+	@$(BD)/TestSource
+	@echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..."
+	@$(BD)/TestHttp
 	@echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..."
 	@$(BD)/TestOob
 	$(BD)/TestOob send 2 > $(BD)/oob_send.d2s && $(BD)/TestOob read $(BD)/oob_send.d2s
@@ -397,13 +405,26 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS)
 	test/das3_csv_test.sh $(BD)
 	test/das3_info_test.sh $(BD)
 	test/das3_from_das2_test.sh $(BD)
+	test/das_fetch_test.sh $(BD)            # test/das_fetch/*.json nodes -> query URLs, offline
 	@echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv"
 	$(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv
 	@echo "INFO: ==============================================="
 	@echo "INFO: All core test programs completed without errors"
 	@echo "INFO: ==============================================="
 
 # Stretch fixtures, see Linux.mak.  This target never fails the build on its own.
+# Tests that need more than a C toolchain and bash.  Not in 'make test', so
+# the base suite runs anywhere the library builds; run these by hand or in
+# a job that has the extras.
+#
+#   das_cache_test.sh  node cache revalidation against a local HTTP server
+#                      written in python3: a second run makes no request,
+#                      an aged copy is renewed by a 304, a changed node is
+#                      fetched again, an unreachable server leaves the old
+#                      copy in use
+test_more: $(BD)/das_fetch
+	test/das_cache_test.sh $(BD)
+
 .PHONY: future
 future: $(BD) $(BD)/TestV3Read
 	@echo "INFO: Stretch fixtures we don't expect to read yet (test/streams/notimp_*):"

buildfiles/Emscripten.mak

@@ -22,8 +22,10 @@
 #
 # What is left out, and why:
 #
-#   http.c credentials.c node.c   sockets and OpenSSL do not exist in a
-#                                 browser; fetch() on the JS side replaces them
+#   http.c credentials.c catalog.c   sockets, OpenSSL and the node cache do
+#                                 not exist in a browser.  fetch() on the JS
+#                                 side drives DasNodeWalk instead; node.c,
+#                                 source.c and req.c never touch a transport
 #   dft.c                         FFTW is not built for wasm yet.  DFT=yes
 #                                 puts it back given FFTW_INC.
 #   cli.c send.c                  program scaffolding, nothing a library
@@ -50,7 +52,7 @@ TARG=libdas3
 
 include buildfiles/sources.mak
 
-NET_SRCS:=das3/http.c das3/credentials.c das3/node.c
+NET_SRCS:=das3/http.c das3/credentials.c das3/catalog.c
 PROG_SRCS:=das3/cli.c das3/send.c
 DFT_SRCS:=das3/dft.c
 
@@ -81,7 +83,7 @@ HDRS:=$(notdir $(LIB_HDRS))
 # opens a URL, reads credentials, or needs SPICE stays out.
 TEST_PROGS:=TestValue TestDatum TestGen TestForm TestCplx TestVar \
  TestVarSubset TestDim TestDesc TestUnits TestTT2000 TestArray TestDs TestIter \
- TestRaggedEncode TestFilter TestUri TestV3Read TestFeed
+ TestRaggedEncode TestFilter TestUri TestV3Read TestFeed TestWalk
 
 ##############################################################################
 # Build definitions

buildfiles/Linux.mak

@@ -181,7 +181,7 @@ $(DESTDIR)$(INST_NAT_BIN)/%:$(BD)/%
 # archive.  Listing those too would let a missing one, say the object for a
 # newly added source, go unbuilt while the archive looks up to date.
 .SECONDARY: $(UTIL_OBJS)
-.PHONY: test build
+.PHONY: test test_more build
 
 ## Explicit Rules  ###########################################################
 
@@ -372,6 +372,14 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS)
 	$(BD)/TestUri $(BD)/uri_tplt
 	@echo "INFO: Running unit test for credentials manager, $(BD)/TestCredMngr..."
 	@$(BD)/TestCredMngr $(BD)
+	@echo "INFO: Running unit test for catalog walks over local nodes, $(BD)/TestWalk..."
+	@$(BD)/TestWalk
+	@echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..."
+	@$(BD)/TestCat
+	@echo "INFO: Running unit test for source descriptions and requisitions, $(BD)/TestSource..."
+	@$(BD)/TestSource
+	@echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..."
+	@$(BD)/TestHttp
 	@echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..."
 	@$(BD)/TestOob
 	$(BD)/TestOob send 2 > $(BD)/oob_send.d2s && $(BD)/TestOob read $(BD)/oob_send.d2s
@@ -397,6 +405,7 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS)
 	test/das3_csv_test.sh $(BD)             # examples/ex23_tracers_mag_hsk.d3b -> .csv
 	test/das3_info_test.sh $(BD)            # examples/ex*.info golds for the dataset printers
 	test/das3_from_das2_test.sh $(BD)       # examples/ex11 pass-through, metafile gold, reject
+	test/das_fetch_test.sh $(BD)            # test/das_fetch/*.json nodes -> query URLs, offline
 	@echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv"
 	$(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv
 	@echo "INFO: ==============================================="
@@ -410,6 +419,18 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS)
 # expect them to fail today.  Kept out of 'test' (which must stay green) and parked here
 # so you can watch for one starting to pass -- that's the cue to promote it out of the
 # notimp_ prefix into the regular suite.  This target never fails the build on its own.
+# Tests that need more than a C toolchain and bash.  Not in 'make test', so
+# the base suite runs anywhere the library builds; run these by hand or in
+# a job that has the extras.
+#
+#   das_cache_test.sh  node cache revalidation against a local HTTP server
+#                      written in python3: a second run makes no request,
+#                      an aged copy is renewed by a 304, a changed node is
+#                      fetched again, an unreachable server leaves the old
+#                      copy in use
+test_more: $(BD)/das_fetch
+	test/das_cache_test.sh $(BD)
+
 .PHONY: future
 future: $(BD) $(BD)/TestV3Read
 	@echo "INFO: Stretch fixtures we don't expect to read yet (test/streams/notimp_*):"

buildfiles/Windows.mak

@@ -388,6 +388,14 @@ test_main:
 	$(BD)\TestUri.exe $(BD)\uri_tplt
 	@echo INFO: Running unit test for credentials manager, $(BD)\TestCredMngr...
 	@$(BD)\TestCredMngr.exe $(BD)
+	@echo INFO: Running unit test for catalog walks over local nodes, $(BD)\TestWalk...
+	@$(BD)\TestWalk.exe
+	@echo INFO: Running unit test for catalog sessions over local nodes, $(BD)\TestCat...
+	@$(BD)\TestCat.exe
+	@echo INFO: Running unit test for source descriptions and requisitions, $(BD)\TestSource...
+	@$(BD)\TestSource.exe
+	@echo INFO: Running unit test for the URL parser, $(BD)\TestHttp...
+	@$(BD)\TestHttp.exe
 	@echo INFO: Running unit test for exception names and send.h writers, $(BD)\TestOob...
 	@$(BD)\TestOob.exe
 	$(BD)\TestOob.exe send 2 > $(BD)\oob_send.d2s
@@ -398,6 +406,8 @@ test_main:
 	@$(BD)\TestCompat.exe $(BD)
 	@echo INFO: Running unit test for CSVs with variable length item data, $(BD)\das3_csv
 	$(BD)\das3_csv.exe < examples\ex21_tracers_cdpu_status.d3b > $(BD)\ex21_tracers_cdpu_status.csv
+	@echo INFO: Running offline catalog node to query URL check, $(BD)\das_fetch...
+	$(BD)\das_fetch.exe -C none url test\das_fetch\geomag_flex_0.7.json 0
 	@echo INFO: ===============================================
 	@echo INFO: All core test programs completed without errors
 	@echo INFO: ===============================================

buildfiles/cmake/das2CTests.cmake

@@ -64,6 +64,10 @@ _das_test(TestRaggedEncode main TestRaggedEncode
 _das_test(TestFilter main TestFilter)
 _das_test(TestUri main TestUri "${_bd}/uri_tplt")
 _das_test(TestCredMngr main TestCredMngr "${_bd}")
+_das_test(TestWalk main TestWalk)
+_das_test(TestCat main TestCat)
+_das_test(TestSource main TestSource)
+_das_test(TestHttp main TestHttp)
 _das_test(TestOob main TestOob)
 foreach(_v 2 3)
 	if(_v EQUAL 2)
@@ -83,6 +87,15 @@ _das_redir(das3_csv_varlen main
 	$<TARGET_FILE:das3_csv>
 )
 
+# das_fetch: a saved catalog node and its example 0 become a query URL, no
+# network.  The full offline suite is test/das_fetch_test.sh (bash only);
+# test/das_cache_test.sh covers node cache revalidation (bash and python3)
+# and is outside the base suite, see 'make test_more' in the make files.
+_das_test(das_fetch_url main das_fetch -C none url test/das_fetch/geomag_flex_0.7.json 0)
+set_tests_properties(das_fetch_url PROPERTIES PASS_REGULAR_EXPRESSION
+	"read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60"
+)
+
 # test_spice #################################################################
 
 if(DAS_WITH_SPICE)

buildfiles/sources.mak

@@ -15,6 +15,7 @@ LIB_SRCS = das3/das1.c das3/array.c das3/buffer.c das3/builder.c das3/cli.c \
  das3/dataset_hdr2.c das3/dataset_hdr3.c das3/datum.c das3/descriptor.c \
  das3/dft.c das3/dimension.c das3/dsdf.c das3/encoding.c das3/http.c \
  das3/io.c das3/iterator.c das3/json.c das3/toml.c das3/log.c das3/node.c \
+ das3/catalog.c das3/req.c das3/source.c \
  das3/oob.c das3/operator.c das3/packet.c das3/plane.c das3/processor.c \
  das3/property.c das3/send.c das3/stream.c das3/time.c das3/tt2000.c \
  das3/units.c das3/utf8.c das3/util.c das3/value.c \
@@ -29,7 +30,7 @@ LIB_HDRS = das3/defs.h das3/time.h das3/das1.h das3/util.h das3/log.h \
  das3/dimension.h das3/dataset.h das3/plane.h das3/packet.h das3/stream.h \
  das3/processor.h das3/property.h das3/oob.h das3/io.h das3/iterator.h \
  das3/builder.h das3/dsdf.h das3/credentials.h das3/http.h das3/dft.h \
- das3/json.h das3/toml.h das3/node.h das3/cli.h das3/send.h das3/uri.h \
+ das3/json.h das3/toml.h das3/node.h das3/catalog.h das3/req.h das3/source.h das3/cli.h das3/send.h das3/uri.h \
  das3/codec.h das3/codex.h das3/core.h das3/variable.h \
  das3/form.h das3/form_rot.h das3/form_lin.h das3/form_pt.h \
  das3/form_cplx.h das3/form_vec.h das3/form_pos.h das3/das2_compat.h \
@@ -50,16 +51,17 @@ UTIL_SRCS = utilities/das1_inctime.c utilities/das2_prtime.c \
  utilities/das2_from_das1.c utilities/das2_from_tagged_das1.c \
  utilities/das1_ascii.c utilities/das1_bin_avg.c utilities/das2_bin_ratesec.c \
  utilities/das2_psd.c utilities/das2_hapi.c utilities/das2_histo.c \
- utilities/das2_cache_rdr.c utilities/das3_node.c utilities/das3_csv.c \
- utilities/das3_info.c utilities/das3_text.c utilities/das3_from_das2.c
+ utilities/das2_cache_rdr.c utilities/das3_csv.c \
+ utilities/das3_info.c utilities/das3_text.c utilities/das3_from_das2.c \
+ utilities/das_fetch.c
 
 SPICE_UTIL_SRCS = utilities/das3_spice.c
 
 CDF_UTIL_SRCS = utilities/das3_cdf.c utilities/das3_from_cdf.c \
  utilities/das2_from_cdf.c
 
 TEST_SRCS = test/TestUnits.c test/TestArray.c test/TestDs.c test/TestBuilder.c \
- test/TestAuth.c test/TestCatalog.c test/TestTT2000.c test/ex_das_cli.c \
+ test/TestAuth.c test/TestCatalog.c test/TestWalk.c test/TestCat.c test/TestSource.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \
  test/ex_das_ephem.c test/TestCredMngr.c test/TestV3Read.c test/TestFeed.c \
  test/TestProp.c \
  test/TestIter.c test/TestUri.c test/TestFilter.c test/TestValue.c \