diff --git a/.gitignore b/.gitignore index 35b87b7..969705e 100644 --- a/.gitignore +++ b/.gitignore @@ -12,5 +12,5 @@ test/test_nan tmp/* CLAUDE.md .claude -co_notes +c*_notes diff --git a/README.md b/README.md index ce12b7e..719b6c6 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,13 @@ to stdout after all input is complete. All the rest: process data packet by packet and output data as soon as it's ready. +The exception at the front of a pipeline is **das_fetch**. It takes no +stream input. Give it a das catalog path and it walks the federated catalog, +works out the server URL and query string from the data source definition, +and streams the response body to standard output, whatever format that is. +Run `das_fetch node uiowa` to start browsing, or `das_fetch -h` for the +query key rules. + ## Reporting bugs Please use the issue tracker for the [das2C](https://github.com/das-developers/das2C/issues) github.com project to report any problems with the library. If you've fixed a bug, diff --git a/buildfiles/Darwin.mak b/buildfiles/Darwin.mak index d3c1bf9..5178be9 100644 --- a/buildfiles/Darwin.mak +++ b/buildfiles/Darwin.mak @@ -217,7 +217,7 @@ $(DESTDIR)$(INST_NAT_BIN)/%:$(BD)/% # archive. Listing those too would let a missing one, say the object for a # newly added source, go unbuilt while the archive looks up to date. .SECONDARY: $(UTIL_OBJS) -.PHONY: test build +.PHONY: test test_more build ## Explicit Rules ########################################################### @@ -374,6 +374,14 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) $(BD)/TestUri $(BD)/uri_tplt @echo "INFO: Running unit test for credentials manager, $(BD)/TestCredMngr..." @$(BD)/TestCredMngr $(BD) + @echo "INFO: Running unit test for catalog walks over local nodes, $(BD)/TestWalk..." + @$(BD)/TestWalk + @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." + @$(BD)/TestCat + @echo "INFO: Running unit test for source descriptions and requisitions, $(BD)/TestSource..." + @$(BD)/TestSource + @echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..." + @$(BD)/TestHttp @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." @$(BD)/TestOob $(BD)/TestOob send 2 > $(BD)/oob_send.d2s && $(BD)/TestOob read $(BD)/oob_send.d2s @@ -397,6 +405,7 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) test/das3_csv_test.sh $(BD) test/das3_info_test.sh $(BD) test/das3_from_das2_test.sh $(BD) + test/das_fetch_test.sh $(BD) # test/das_fetch/*.json nodes -> query URLs, offline @echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv" $(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv @echo "INFO: ===============================================" @@ -404,6 +413,18 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BULID_UTIL_PROGS) @echo "INFO: ===============================================" # Stretch fixtures, see Linux.mak. This target never fails the build on its own. +# Tests that need more than a C toolchain and bash. Not in 'make test', so +# the base suite runs anywhere the library builds; run these by hand or in +# a job that has the extras. +# +# das_cache_test.sh node cache revalidation against a local HTTP server +# written in python3: a second run makes no request, +# an aged copy is renewed by a 304, a changed node is +# fetched again, an unreachable server leaves the old +# copy in use +test_more: $(BD)/das_fetch + test/das_cache_test.sh $(BD) + .PHONY: future future: $(BD) $(BD)/TestV3Read @echo "INFO: Stretch fixtures we don't expect to read yet (test/streams/notimp_*):" diff --git a/buildfiles/Emscripten.mak b/buildfiles/Emscripten.mak index d8422e1..f27d636 100644 --- a/buildfiles/Emscripten.mak +++ b/buildfiles/Emscripten.mak @@ -22,8 +22,10 @@ # # What is left out, and why: # -# http.c credentials.c node.c sockets and OpenSSL do not exist in a -# browser; fetch() on the JS side replaces them +# http.c credentials.c catalog.c sockets, OpenSSL and the node cache do +# not exist in a browser. fetch() on the JS +# side drives DasNodeWalk instead; node.c, +# source.c and req.c never touch a transport # dft.c FFTW is not built for wasm yet. DFT=yes # puts it back given FFTW_INC. # cli.c send.c program scaffolding, nothing a library @@ -50,7 +52,7 @@ TARG=libdas3 include buildfiles/sources.mak -NET_SRCS:=das3/http.c das3/credentials.c das3/node.c +NET_SRCS:=das3/http.c das3/credentials.c das3/catalog.c PROG_SRCS:=das3/cli.c das3/send.c DFT_SRCS:=das3/dft.c @@ -81,7 +83,7 @@ HDRS:=$(notdir $(LIB_HDRS)) # opens a URL, reads credentials, or needs SPICE stays out. TEST_PROGS:=TestValue TestDatum TestGen TestForm TestCplx TestVar \ TestVarSubset TestDim TestDesc TestUnits TestTT2000 TestArray TestDs TestIter \ - TestRaggedEncode TestFilter TestUri TestV3Read TestFeed + TestRaggedEncode TestFilter TestUri TestV3Read TestFeed TestWalk ############################################################################## # Build definitions diff --git a/buildfiles/Linux.mak b/buildfiles/Linux.mak index 112f674..d3d08f9 100644 --- a/buildfiles/Linux.mak +++ b/buildfiles/Linux.mak @@ -181,7 +181,7 @@ $(DESTDIR)$(INST_NAT_BIN)/%:$(BD)/% # archive. Listing those too would let a missing one, say the object for a # newly added source, go unbuilt while the archive looks up to date. .SECONDARY: $(UTIL_OBJS) -.PHONY: test build +.PHONY: test test_more build ## Explicit Rules ########################################################### @@ -372,6 +372,14 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) $(BD)/TestUri $(BD)/uri_tplt @echo "INFO: Running unit test for credentials manager, $(BD)/TestCredMngr..." @$(BD)/TestCredMngr $(BD) + @echo "INFO: Running unit test for catalog walks over local nodes, $(BD)/TestWalk..." + @$(BD)/TestWalk + @echo "INFO: Running unit test for catalog sessions over local nodes, $(BD)/TestCat..." + @$(BD)/TestCat + @echo "INFO: Running unit test for source descriptions and requisitions, $(BD)/TestSource..." + @$(BD)/TestSource + @echo "INFO: Running unit test for the URL parser, $(BD)/TestHttp..." + @$(BD)/TestHttp @echo "INFO: Running unit test for exception names and send.h writers, $(BD)/TestOob..." @$(BD)/TestOob $(BD)/TestOob send 2 > $(BD)/oob_send.d2s && $(BD)/TestOob read $(BD)/oob_send.d2s @@ -397,6 +405,7 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) test/das3_csv_test.sh $(BD) # examples/ex23_tracers_mag_hsk.d3b -> .csv test/das3_info_test.sh $(BD) # examples/ex*.info golds for the dataset printers test/das3_from_das2_test.sh $(BD) # examples/ex11 pass-through, metafile gold, reject + test/das_fetch_test.sh $(BD) # test/das_fetch/*.json nodes -> query URLs, offline @echo "INFO: Running unit test for CSVs with variable length item data, $(BD)/das3_csv" $(BD)/das3_csv < examples/ex21_tracers_cdpu_status.d3b > $(BD)/ex21_tracers_cdpu_status.csv @echo "INFO: ===============================================" @@ -410,6 +419,18 @@ test_main: $(BD) $(BD)/$(TARG).a $(BUILD_TEST_PROGS) $(BUILD_UTIL_PROGS) # expect them to fail today. Kept out of 'test' (which must stay green) and parked here # so you can watch for one starting to pass -- that's the cue to promote it out of the # notimp_ prefix into the regular suite. This target never fails the build on its own. +# Tests that need more than a C toolchain and bash. Not in 'make test', so +# the base suite runs anywhere the library builds; run these by hand or in +# a job that has the extras. +# +# das_cache_test.sh node cache revalidation against a local HTTP server +# written in python3: a second run makes no request, +# an aged copy is renewed by a 304, a changed node is +# fetched again, an unreachable server leaves the old +# copy in use +test_more: $(BD)/das_fetch + test/das_cache_test.sh $(BD) + .PHONY: future future: $(BD) $(BD)/TestV3Read @echo "INFO: Stretch fixtures we don't expect to read yet (test/streams/notimp_*):" diff --git a/buildfiles/Windows.mak b/buildfiles/Windows.mak index 8b83108..d2d7fa4 100644 --- a/buildfiles/Windows.mak +++ b/buildfiles/Windows.mak @@ -388,6 +388,14 @@ test_main: $(BD)\TestUri.exe $(BD)\uri_tplt @echo INFO: Running unit test for credentials manager, $(BD)\TestCredMngr... @$(BD)\TestCredMngr.exe $(BD) + @echo INFO: Running unit test for catalog walks over local nodes, $(BD)\TestWalk... + @$(BD)\TestWalk.exe + @echo INFO: Running unit test for catalog sessions over local nodes, $(BD)\TestCat... + @$(BD)\TestCat.exe + @echo INFO: Running unit test for source descriptions and requisitions, $(BD)\TestSource... + @$(BD)\TestSource.exe + @echo INFO: Running unit test for the URL parser, $(BD)\TestHttp... + @$(BD)\TestHttp.exe @echo INFO: Running unit test for exception names and send.h writers, $(BD)\TestOob... @$(BD)\TestOob.exe $(BD)\TestOob.exe send 2 > $(BD)\oob_send.d2s @@ -398,6 +406,8 @@ test_main: @$(BD)\TestCompat.exe $(BD) @echo INFO: Running unit test for CSVs with variable length item data, $(BD)\das3_csv $(BD)\das3_csv.exe < examples\ex21_tracers_cdpu_status.d3b > $(BD)\ex21_tracers_cdpu_status.csv + @echo INFO: Running offline catalog node to query URL check, $(BD)\das_fetch... + $(BD)\das_fetch.exe -C none url test\das_fetch\geomag_flex_0.7.json 0 @echo INFO: =============================================== @echo INFO: All core test programs completed without errors @echo INFO: =============================================== diff --git a/buildfiles/cmake/das2CTests.cmake b/buildfiles/cmake/das2CTests.cmake index 35b4ea2..e02ba14 100644 --- a/buildfiles/cmake/das2CTests.cmake +++ b/buildfiles/cmake/das2CTests.cmake @@ -64,6 +64,10 @@ _das_test(TestRaggedEncode main TestRaggedEncode _das_test(TestFilter main TestFilter) _das_test(TestUri main TestUri "${_bd}/uri_tplt") _das_test(TestCredMngr main TestCredMngr "${_bd}") +_das_test(TestWalk main TestWalk) +_das_test(TestCat main TestCat) +_das_test(TestSource main TestSource) +_das_test(TestHttp main TestHttp) _das_test(TestOob main TestOob) foreach(_v 2 3) if(_v EQUAL 2) @@ -83,6 +87,15 @@ _das_redir(das3_csv_varlen main $ ) +# das_fetch: a saved catalog node and its example 0 become a query URL, no +# network. The full offline suite is test/das_fetch_test.sh (bash only); +# test/das_cache_test.sh covers node cache revalidation (bash and python3) +# and is outside the base suite, see 'make test_more' in the make files. +_das_test(das_fetch_url main das_fetch -C none url test/das_fetch/geomag_flex_0.7.json 0) +set_tests_properties(das_fetch_url PROPERTIES PASS_REGULAR_EXPRESSION + "read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +) + # test_spice ################################################################# if(DAS_WITH_SPICE) diff --git a/buildfiles/sources.mak b/buildfiles/sources.mak index 830a2c4..f9a3178 100644 --- a/buildfiles/sources.mak +++ b/buildfiles/sources.mak @@ -15,6 +15,7 @@ LIB_SRCS = das3/das1.c das3/array.c das3/buffer.c das3/builder.c das3/cli.c \ das3/dataset_hdr2.c das3/dataset_hdr3.c das3/datum.c das3/descriptor.c \ das3/dft.c das3/dimension.c das3/dsdf.c das3/encoding.c das3/http.c \ das3/io.c das3/iterator.c das3/json.c das3/toml.c das3/log.c das3/node.c \ + das3/catalog.c das3/req.c das3/source.c \ das3/oob.c das3/operator.c das3/packet.c das3/plane.c das3/processor.c \ das3/property.c das3/send.c das3/stream.c das3/time.c das3/tt2000.c \ das3/units.c das3/utf8.c das3/util.c das3/value.c \ @@ -29,7 +30,7 @@ LIB_HDRS = das3/defs.h das3/time.h das3/das1.h das3/util.h das3/log.h \ das3/dimension.h das3/dataset.h das3/plane.h das3/packet.h das3/stream.h \ das3/processor.h das3/property.h das3/oob.h das3/io.h das3/iterator.h \ das3/builder.h das3/dsdf.h das3/credentials.h das3/http.h das3/dft.h \ - das3/json.h das3/toml.h das3/node.h das3/cli.h das3/send.h das3/uri.h \ + das3/json.h das3/toml.h das3/node.h das3/catalog.h das3/req.h das3/source.h das3/cli.h das3/send.h das3/uri.h \ das3/codec.h das3/codex.h das3/core.h das3/variable.h \ das3/form.h das3/form_rot.h das3/form_lin.h das3/form_pt.h \ das3/form_cplx.h das3/form_vec.h das3/form_pos.h das3/das2_compat.h \ @@ -50,8 +51,9 @@ UTIL_SRCS = utilities/das1_inctime.c utilities/das2_prtime.c \ utilities/das2_from_das1.c utilities/das2_from_tagged_das1.c \ utilities/das1_ascii.c utilities/das1_bin_avg.c utilities/das2_bin_ratesec.c \ utilities/das2_psd.c utilities/das2_hapi.c utilities/das2_histo.c \ - utilities/das2_cache_rdr.c utilities/das3_node.c utilities/das3_csv.c \ - utilities/das3_info.c utilities/das3_text.c utilities/das3_from_das2.c + utilities/das2_cache_rdr.c utilities/das3_csv.c \ + utilities/das3_info.c utilities/das3_text.c utilities/das3_from_das2.c \ + utilities/das_fetch.c SPICE_UTIL_SRCS = utilities/das3_spice.c @@ -59,7 +61,7 @@ CDF_UTIL_SRCS = utilities/das3_cdf.c utilities/das3_from_cdf.c \ utilities/das2_from_cdf.c TEST_SRCS = test/TestUnits.c test/TestArray.c test/TestDs.c test/TestBuilder.c \ - test/TestAuth.c test/TestCatalog.c test/TestTT2000.c test/ex_das_cli.c \ + test/TestAuth.c test/TestCatalog.c test/TestWalk.c test/TestCat.c test/TestSource.c test/TestHttp.c test/TestTT2000.c test/ex_das_cli.c \ test/ex_das_ephem.c test/TestCredMngr.c test/TestV3Read.c test/TestFeed.c \ test/TestProp.c \ test/TestIter.c test/TestUri.c test/TestFilter.c test/TestValue.c \ diff --git a/das3/catalog.c b/das3/catalog.c new file mode 100644 index 0000000..cbb49c5 --- /dev/null +++ b/das3/catalog.c @@ -0,0 +1,985 @@ +/* Copyright (C) 2026 Chris Piker + * + * Author: C. Piker, via Claude Fable 5.1 + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* A catalog session. Grew up as a pilot inside utilities/das_fetch.c and + * moved here once a second client wanted the same rules for where paths + * start. + * Policy misuse is reported with daslog_error and DASERR_CAT rather than + * das_error, because the usual source of a bad policy is a command line + * argument and a program should get to decide its exit code for that. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "util.h" +#include "log.h" +#include "http.h" +#include "node_priv.h" +#include "source.h" +#include "catalog.h" + +/* Where this session keeps copies of nodes fetched over HTTP and how long + * it trusts them. Empty sDir means no caching; nMaxAgeSec of zero or less + * means always ask the server, see _dascat_fetch(). */ +typedef struct das_node_cache { + char sDir[256]; + int nMaxAgeSec; +} DasNodeCache; + +struct das_cat_mount { + char* sUrl; + char* sUri; +}; + +struct das_cat { + DasCredMngr* pMgr; /* owned */ + char* sAgent; /* owned copy, or NULL for the library default */ + DasNodeCache* pCache; /* owned, NULL until DasCat_setCache() */ + + /* Where paths start. Relative paths get sPrefix in front, or with a + * seed set, the seed's prefix and separator; the two are one knob + * with two setters, and only one may be used. Mounts are consulted + * before the federated catalog for any URI they cover, opportunistic + * grafts rather than a policy. A seed is a mount that turns the + * federated catalog off: the mounts are then the whole catalog, which + * is how an offline catalog is assembled from a handful of files. */ + char* sPrefix; /* never NULL */ + bool bPrefixGiven; /* DasCat_setPrefix() was called */ + struct das_cat_mount* aMounts; /* named nodes loaded by location */ + size_t uMounts; + bool bSeed; /* set by DasCat_setSeed(): the seed and the mounts are the whole catalog */ + char* sSeedUrl; /* owned, where the seed loads from */ + char* sSeedUri; /* owned, its prefix, "" for a root */ + DasNode* pSeed; /* the seed node, loaded on first use */ + + DasNode* pTable; /* the mount table, built on first use */ + DasNode* pDas2; /* the compiled in seed, loaded on first use; the + tree below it is the session's memory of the + federated catalog */ + DasNode** aOwned; /* every tree top the session holds, freed with it */ + size_t uOwned; + size_t uOwnedCap; + + char* sLastPath; /* the caller's path to the last resolve, or NULL */ + char* sLastUri; /* that path as the policy understood it */ + char* sNear; /* after a failed walk, the deepest container it searched */ + char* sNearRest; /* and the path below it, see DasCat_nearest() */ + DasNode* pNearNode; /* that container when it is a root and so has no URI to + resolve again: the walk's start node, borrowed */ +}; + +/* Defined further down, used throughout */ +static char* _dascat_strdupz(const char* s); +static DasNode* _dascat_loadUrl( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache +); + +/* ************************************************************************* */ +/* Loading nodes: the native driver + * + * node.c doesn't read files or open sockets. It parses the documents it + * is handed and, through DasNodeWalk, says which document it needs next. + * What follows is the driver for programs on a real machine: it reads + * files, keeps the node cache, makes the HTTP requests, and loops a walk + * until it stops. A client that fetches for itself, a browser build in + * particular, just use node.h alone and exclude this file. */ + +/* ************************************************************************* */ +/* Local file nodes. + * + * Catalog nodes are not only served over HTTP. Test fixtures, detached + * catalogs under development, and 'file://' URLs handed over by other + * tools all want a node read straight from disk. Anything that is not an http(s) URL + * is treated as a path; the "file://" prefix is accepted and stripped. + * Only the path forms "file:///abs/path" and "file://rel/path" are handled, + * host names in file URLs are not. */ + +static bool _dascat_isFileUrl(const char* sUrl) +{ + if(sUrl == NULL) return false; + if(strncmp(sUrl, "http://", 7) == 0) return false; + if(strncmp(sUrl, "https://", 8) == 0) return false; + return true; +} + +static DasAry* _dascat_readFile(const char* sUrl) +{ + const char* sPath = sUrl; + if(strncmp(sUrl, "file://", 7) == 0) sPath += 7; + + FILE* pFile = fopen(sPath, "rb"); + if(pFile == NULL){ + daslog_warn_v("Couldn't open node file %s, %s", sPath, strerror(errno)); + return NULL; + } + + DasAry* pAry = new_DasAry("node_file", vtUByte, 1, NULL, RANK_1(0), UNIT_DIMENSIONLESS); + char buf[16384]; + size_t uRead = 0; + while((uRead = fread(buf, 1, sizeof(buf), pFile)) > 0){ + if(!DasAry_append(pAry, (const ubyte*)buf, uRead)){ + dec_DasAry(pAry); + fclose(pFile); + return NULL; + } + } + bool bErr = (ferror(pFile) != 0); + fclose(pFile); + if(bErr){ + daslog_warn_v("Error reading node file %s", sPath); + dec_DasAry(pAry); + return NULL; + } + return pAry; +} + +/* ************************************************************************* */ +/* Node cache */ + +/* Only a DasCat has cache settings, nodes made through the sessionless + * constructors in node.h aren't cached. One knob, per session, instead + * of a second process wide one to confuse it with. */ + +/* Cache file name for a URL: the URL minus its scheme with everything + * outside [A-Za-z0-9.-] turned into '_', then a hash of the whole URL so + * that two URLs which sanitize alike still get separate files. Readable + * names make an 'ls' of the cache directory useful when debugging a + * catalog, a bare hash would not. */ +static bool _dascat_cachePath( + const DasNodeCache* pCache, const char* sUrl, char* sPath, size_t uLen +){ + if((pCache == NULL)||(pCache->sDir[0] == '\0')) return false; + + uint32_t uHash = 2166136261u; /* FNV-1a, no need for anything stronger */ + for(const char* p = sUrl; *p != '\0'; ++p){ + uHash ^= (unsigned char)(*p); + uHash *= 16777619u; + } + + const char* p = sUrl; + if(strncmp(p, "https://", 8) == 0) p += 8; + else if(strncmp(p, "http://", 7) == 0) p += 7; + + char sName[160]; + size_t u = 0; + for(; (*p != '\0') && (u < sizeof(sName) - 1); ++p){ + char c = *p; + sName[u++] = (isalnum((unsigned char)c)||(c == '.')||(c == '-')) ? c : '_'; + } + sName[u] = '\0'; + + int n = snprintf(sPath, uLen, "%s%s%s_%08" PRIx32 ".json", pCache->sDir, DAS_DSEPS, sName, uHash); + return (n > 0) && ((size_t)n < uLen); +} + +/* A fresh enough cached copy, or NULL */ +static DasAry* _dascat_readCache(const DasNodeCache* pCache, const char* sPath) +{ + if((pCache == NULL)||(pCache->nMaxAgeSec <= 0)) return NULL; + struct stat st; + if(stat(sPath, &st) != 0) return NULL; + if((time(NULL) - st.st_mtime) > pCache->nMaxAgeSec) return NULL; + return _dascat_readFile(sPath); +} + +/* Written to a side file then renamed, so a reader never sees a partial + * document when two programs walk the catalog at once. Failures only cost + * the cache, so they are logged at debug level and otherwise ignored. */ +static void _dascat_writeCache(const char* sPath, const char* pBytes, size_t uBytes) +{ + char sTmp[600]; + if(snprintf(sTmp, sizeof(sTmp), "%s.%ld.tmp", sPath, (long)time(NULL)) >= (int)sizeof(sTmp)) return; + + if(das_mkdirsto(sPath) != DAS_OKAY){ + daslog_debug_v("Couldn't create node cache directory for %s", sPath); + return; + } + FILE* pFile = fopen(sTmp, "wb"); + if(pFile == NULL){ + daslog_debug_v("Couldn't write node cache file %s, %s", sTmp, strerror(errno)); + return; + } + bool bOk = (fwrite(pBytes, 1, uBytes, pFile) == uBytes); + bOk = (fclose(pFile) == 0) && bOk; + if(!bOk || (das_rename(sTmp, sPath) != 0)){ + daslog_debug_v("Couldn't finish node cache file %s", sPath); + remove(sTmp); + } +} + +/* The validators a cached node came with live beside it in a .hdr side + * file, at most two lines, the ETag and Last-Modified header values + * verbatim. When the copy is older than the session trusts both go back + * as If-None-Match and If-Modified-Since, and a 304 renews the copy for + * one round trip and no body. Both are kept because servers differ: the + * das2.org tree sends both, GitHub raw sends ETag only, dasflex today + * sends neither. The date is never derived from the cache file's + * mtime, which renewal moves forward. */ +static void _dascat_hdrPath(const char* sCache, char* sHdr, size_t uLen) +{ + snprintf(sHdr, uLen, "%s.hdr", sCache); +} + +/* Request header lines for the cached copy at sCache, "" if it has none */ +static void _dascat_readValidators(const char* sCache, char* sReq, size_t uLen) +{ + sReq[0] = '\0'; + char sHdr[600]; + _dascat_hdrPath(sCache, sHdr, sizeof(sHdr)); + FILE* pFile = fopen(sHdr, "r"); + if(pFile == NULL) return; + + char sLine[1024]; + size_t u = 0; + while(fgets(sLine, sizeof(sLine), pFile) != NULL){ + size_t uLine = strlen(sLine); + while((uLine > 0) && ((sLine[uLine-1] == '\n')||(sLine[uLine-1] == '\r'))) + sLine[--uLine] = '\0'; + const char* sName = NULL; + const char* sVal = NULL; + if(strncmp(sLine, "ETag: ", 6) == 0){ sName = "If-None-Match"; sVal = sLine + 6; } + else if(strncmp(sLine, "Last-Modified: ", 15) == 0){ sName = "If-Modified-Since"; sVal = sLine + 15; } + if((sName == NULL)||(sVal[0] == '\0')) continue; + int n = snprintf(sReq + u, uLen - u, "%s: %s\r\n", sName, sVal); + if((n < 0)||((size_t)n >= uLen - u)){ sReq[u] = '\0'; break; } + u += (size_t)n; + } + fclose(pFile); +} + +/* Save the validators a 200 carried, or drop the side file if it carried + * none, so a validator from an older body is never sent for a newer one */ +static void _dascat_writeValidators(const char* sCache, const DasHttpResp* pRes) +{ + char sHdr[600]; + _dascat_hdrPath(sCache, sHdr, sizeof(sHdr)); + char sETag[256] = {'\0'}; + char sMod[128] = {'\0'}; + bool bETag = DasHttpResp_header(pRes, "ETag", sETag, sizeof(sETag)); + bool bMod = DasHttpResp_header(pRes, "Last-Modified", sMod, sizeof(sMod)); + if(!bETag && !bMod){ remove(sHdr); return; } + + FILE* pFile = fopen(sHdr, "w"); + if(pFile == NULL){ + daslog_debug_v("Couldn't write node cache side file %s, %s", sHdr, strerror(errno)); + return; + } + if(bETag) fprintf(pFile, "ETag: %s\n", sETag); + if(bMod) fprintf(pFile, "Last-Modified: %s\n", sMod); + fclose(pFile); +} + +/* ************************************************************************* */ +/* One document, from wherever it lives */ + +struct dascat_fetch { + DasAry* pBytes; /* the document, NULL when nothing came back */ + char sFinal[512]; /* the URL after redirects, for messages */ + char sCache[512]; /* where a copy goes, "" when not cached */ + bool bFromCache; /* copy used as is, nothing to write back */ + bool bRenewed; /* server confirmed the copy, rewrite it to move its age */ + bool bNoAnswer; /* no HTTP status came back: refused, timed out, no route */ + DasHttpResp res; +}; + +/* Calls das_error only when memory runs out, network operations are + * assumed to fail sometimes. The 20 MB limit is a sanity check, no + * catalog node comes near it. */ +static void _dascat_fetch( + DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache, + const char* sUrl, float rConSec, struct dascat_fetch* pF +){ + memset(pF, 0, sizeof(*pF)); + pF->res.nCode = -1; + + if(_dascat_isFileUrl(sUrl)){ + pF->pBytes = _dascat_readFile(sUrl); + strncpy(pF->sFinal, sUrl, sizeof(pF->sFinal) - 1); + return; + } + if(_dascat_cachePath(pCache, sUrl, pF->sCache, sizeof(pF->sCache)) && + ((pF->pBytes = _dascat_readCache(pCache, pF->sCache)) != NULL)){ + daslog_debug_v("Node %s read from cache", sUrl); + strncpy(pF->sFinal, sUrl, sizeof(pF->sFinal) - 1); + pF->bFromCache = true; + return; + } + + /* No copy, or one older than the session trusts. With validators from + * the old copy the request is conditional and a 304 renews it for no + * body bytes. If the server cannot be reached at all the old copy + * still beats nothing: a warm cache keeps working offline. */ + char sReqHdrs[1024] = {'\0'}; + bool bHaveOld = (pF->sCache[0] != '\0') && das_isfile(pF->sCache); + if(bHaveOld) _dascat_readValidators(pF->sCache, sReqHdrs, sizeof(sReqHdrs)); + + pF->pBytes = das_http_readUrl_ex( + sUrl, sAgent, pMgr, &pF->res, 1024*1024*20, rConSec, sReqHdrs + ); + if(pF->pBytes != NULL){ + das_url_toStr(&(pF->res.url), pF->sFinal, sizeof(pF->sFinal)); + return; + } + strncpy(pF->sFinal, sUrl, sizeof(pF->sFinal) - 1); + if(pF->res.nCode == 304){ + daslog_debug_v("Node %s unchanged on the server, cached copy renewed", sUrl); + pF->pBytes = _dascat_readFile(pF->sCache); + pF->bRenewed = true; + } + else if(bHaveOld){ + daslog_warn_v("%s, using the cached copy of %s", + pF->res.sError ? pF->res.sError : "Fetch failed", sUrl); + pF->pBytes = _dascat_readFile(pF->sCache); + pF->bFromCache = true; + } + else{ + daslog_warn_v("%s", pF->res.sError ? pF->res.sError : "Fetch failed"); + pF->bNoAnswer = (pF->res.nCode < 0); + } +} + +/* Keep a copy of a document the walk accepted. Only documents that parse + * are worth keeping, a server error page is not. A renewal rewrites the + * same bytes, which is what moves the copy's age, and keeps the + * validators it already has. */ +static void _dascat_keep(const struct dascat_fetch* pF, const char* pBytes, size_t uBytes) +{ + if((pF->sCache[0] == '\0') || pF->bFromCache) return; + _dascat_writeCache(pF->sCache, pBytes, uBytes); + if(!pF->bRenewed) _dascat_writeValidators(pF->sCache, &pF->res); +} + +static void _dascat_fetchDone(struct dascat_fetch* pF) +{ + DasHttpResp_freeFields(&pF->res); + if(pF->pBytes != NULL) dec_DasAry(pF->pBytes); + pF->pBytes = NULL; +} + +/* ************************************************************************* */ +/* Driving a walk */ + +/* Loop a walk to its end. pbNoAnswer is set when some request got no + * answer at all, the one failure a longer timeout might cure. A path the + * catalog does not have fails through the walk the same way, and waiting + * longer would not change that. */ +static DasNode* _dascat_drive( + DasNodeWalk* pWalk, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache, float rConSec, bool* pbNoAnswer +){ + struct dascat_fetch f; + while(DasNodeWalk_state(pWalk) == das_walk_fetch){ + const char* sUrl = DasNodeWalk_url(pWalk, 0); + _dascat_fetch(pMgr, sAgent, pCache, sUrl, rConSec, &f); + if(f.pBytes != NULL){ + size_t uLen = 0; + const char* p = (const char*)DasAry_getBytesIn(f.pBytes, DIM0, &uLen); + if(DasNodeWalk_feed(pWalk, 0, p, uLen, f.sFinal) == DAS_OKAY) + _dascat_keep(&f, p, uLen); + } + else{ + if(f.bNoAnswer && (pbNoAnswer != NULL)) *pbNoAnswer = true; + DasNodeWalk_fail(pWalk, 0, f.res.sError); + } + _dascat_fetchDone(&f); + } + return DasNodeWalk_result(pWalk); +} + +/* Remember what a failed walk can say, see DasCat_nearest() */ +static void _DasCat_setNear(DasCat* pThis, const DasNodeWalk* pWalk, DasNode* pFrom) +{ + free(pThis->sNear); pThis->sNear = NULL; + free(pThis->sNearRest); pThis->sNearRest = NULL; + pThis->pNearNode = NULL; + const char* sRest = NULL; + const char* sNear = DasNodeWalk_nearest(pWalk, &sRest); + if(sNear == NULL) return; + /* An empty URI is a root, which cannot be resolved again by name; it + can only be the node the walk started from. The mount table is a + root too, but nothing under it that failed is worth pointing at. */ + if(sNear[0] == '\0'){ + if(pFrom == pThis->pTable) return; + pThis->pNearNode = pFrom; + } + else{ + pThis->sNear = das_strdup(sNear); + } + pThis->sNearRest = _dascat_strdupz(sRest); +} + +/* A path below a node, with this session's settings. Children get a + * slower timeout than roots since there is no second copy of the tree to + * fall back to. */ +static DasNode* _DasCat_walkFrom(DasCat* pThis, DasNode* pFrom, const char* sRel) +{ + DasNodeWalk* pWalk = new_DasNodeWalk(pFrom, sRel); + if(pWalk == NULL) return NULL; + DasNode* pNode = _dascat_drive( + pWalk, pThis->pMgr, pThis->sAgent, pThis->pCache, DASHTTP_TO_MIN * DASHTTP_TO_MULTI, NULL + ); + _DasCat_setNear(pThis, pWalk, pFrom); + del_DasNodeWalk(pWalk); + return pNode; +} + +/* The compiled in seed, from whichever of its mirrors answers. Each is + * tried in turn with a short connect timeout and the round is repeated + * with longer ones, but only when some server gave no answer, so one slow + * mirror does not stall the load. The seed is a tree top the caller + * owns; everything in the federated catalog is a walk from it. */ +static DasNode* _dascat_loadSeed(DasCredMngr* pMgr, const char* sAgent, const DasNodeCache* pCache) +{ + float rConSec = DASHTTP_TO_MIN; + while(rConSec <= DASHTTP_TO_MAX){ + DasNodeWalk* pWalk = new_DasNodeWalk_seed(); + if(pWalk == NULL) return NULL; + bool bNoAnswer = false; + DasNode* pRoot = _dascat_drive(pWalk, pMgr, sAgent, pCache, rConSec, &bNoAnswer); + del_DasNodeWalk(pWalk); + if(pRoot != NULL) return pRoot; + if(!bNoAnswer) break; + rConSec *= DASHTTP_TO_MULTI; + } + daslog_error("No mirror of the das catalog index answered"); + return NULL; +} + +static DasNode* _DasCat_keep(DasCat* pThis, DasNode* pRoot); /* below, with the session */ + +/* The seed given to DasCat_setSeed(), loaded once and kept */ +static DasNode* _DasCat_seedNode(DasCat* pThis) +{ + if(pThis->pSeed == NULL) + pThis->pSeed = _DasCat_keep(pThis, _dascat_loadUrl( + pThis->sSeedUrl, pThis->sSeedUri, pThis->pMgr, pThis->sAgent, pThis->pCache + )); + return pThis->pSeed; +} + +/* The session's copy of the compiled in seed, loaded once and kept, as + * das2py has always done. Every federated lookup is then a walk from it + * and the levels walked through stay loaded for the next one. */ +static DasNode* _DasCat_das2(DasCat* pThis) +{ + if(pThis->pDas2 == NULL) + pThis->pDas2 = _DasCat_keep(pThis, _dascat_loadSeed(pThis->pMgr, pThis->sAgent, pThis->pCache)); + return pThis->pDas2; +} + +/* One node from a URL or file, named by the caller or by its location. + * Since they want a specific URL, go for the max timeout. */ +static DasNode* _dascat_loadUrl( + const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent, + const DasNodeCache* pCache +){ + char sUri[600] = {'\0'}; + if((sPathUri != NULL) && (sPathUri[0] != '\0')) _das_uri_expand(sPathUri, sUri, sizeof(sUri)); + + struct dascat_fetch f; + _dascat_fetch(pMgr, sAgent, pCache, sUrl, DASHTTP_TO_MAX, &f); + DasNode* pNode = NULL; + if(f.pBytes != NULL){ + size_t uLen = 0; + const char* p = (const char*)DasAry_getBytesIn(f.pBytes, DIM0, &uLen); + pNode = _dasnode_fromJson(p, uLen, sUrl, f.sFinal, sUri); + if(pNode != NULL){ + pNode->bOwner = true; + _dascat_keep(&f, p, uLen); + } + } + _dascat_fetchDone(&f); + return pNode; +} + +/* ************************************************************************* */ +/* Small helpers */ + +static void _dascat_setStr(char** ps, const char* s) +{ + free(*ps); + *ps = (s != NULL) ? das_strdup(s) : NULL; +} + +/* Shorthands are expanded by node.c as well; doing it here too keeps + * DasCat_lastUri() the full form even when a resolve fails early. */ +static char* _dascat_expandUri(const char* sIn) +{ + char sBuf[600]; + return das_strdup(_das_uri_expand(sIn, sBuf, sizeof(sBuf))); +} + +/* Take ownership of a tree top for the life of the session */ +static DasNode* _DasCat_keep(DasCat* pThis, DasNode* pRoot) +{ + if(pRoot == NULL) return NULL; + if(pThis->uOwned == pThis->uOwnedCap){ + size_t uCap = (pThis->uOwnedCap == 0) ? 8 : 2 * pThis->uOwnedCap; + DasNode** a = (DasNode**)realloc(pThis->aOwned, uCap * sizeof(DasNode*)); + if(a == NULL){ + das_error(DASERR_CAT, "Couldn't grow the session's node list"); + return pRoot; /* leaks the node rather than losing it */ + } + pThis->aOwned = a; + pThis->uOwnedCap = uCap; + } + pThis->aOwned[pThis->uOwned++] = pRoot; + return pRoot; +} + +/* ************************************************************************* */ +/* Construction */ + +DasCat* new_DasCat(DasCredMngr* pMgr, const char* sAgent) +{ + DasCat* pThis = (DasCat*)calloc(1, sizeof(DasCat)); + if(pThis == NULL){ + das_error(DASERR_CAT, "Couldn't allocate a catalog session"); + return NULL; + } + pThis->pMgr = pMgr; + if((sAgent != NULL) && (sAgent[0] != '\0')) pThis->sAgent = das_strdup(sAgent); + pThis->sPrefix = das_strdup(DASCAT_DEF_PREFIX); + + /* Caching is on from the start. A session that does not want it says + * so; one that forgets gets fast catalog walks and at worst an hour old + * node, which is the better default for the GUIs and scripts built on + * this. */ + char sDir[sizeof(((DasNodeCache*)0)->sDir)]; + int nLen = snprintf(sDir, sizeof(sDir), "%s" DAS_DSEPS "cache" DAS_DSEPS "nodes", das_userdir()); + if((nLen > 0) && (nLen < (int)sizeof(sDir))) + DasCat_setCache(pThis, sDir, DASCAT_DEF_CACHE_SEC); + else + daslog_warn_v("User directory %s too long for the node cache, caching is off", das_userdir()); + return pThis; +} + +void del_DasCat(DasCat* pThis) +{ + if(pThis == NULL) return; + + /* Nodes borrow the credentials manager and the cache settings, so they go + * first. The mount table and the seeds are in aOwned like every tree top. */ + for(size_t u = 0; u < pThis->uOwned; ++u) del_DasNode(pThis->aOwned[u]); + free(pThis->aOwned); + if(pThis->pMgr != NULL) del_CredMngr(pThis->pMgr); + free(pThis->pCache); + + free(pThis->sAgent); + free(pThis->sPrefix); + for(size_t u = 0; u < pThis->uMounts; ++u){ + free(pThis->aMounts[u].sUrl); + free(pThis->aMounts[u].sUri); + } + free(pThis->aMounts); + free(pThis->sLastPath); + free(pThis->sLastUri); + free(pThis->sNear); + free(pThis->sNearRest); + free(pThis->sSeedUrl); + free(pThis->sSeedUri); + free(pThis); +} + +DasCredMngr* DasCat_credMngr(const DasCat* pThis){ return pThis->pMgr; } + +const char* DasCat_agent(const DasCat* pThis){ return pThis->sAgent; } + +void DasCat_setCache(DasCat* pThis, const char* sDir, int nMaxAgeSec) +{ + if(pThis->pCache == NULL) + pThis->pCache = (DasNodeCache*)calloc(1, sizeof(DasNodeCache)); + if(pThis->pCache == NULL){ + das_error(DASERR_CAT, "Couldn't allocate the node cache settings"); + return; + } + if((sDir == NULL)||(sDir[0] == '\0')){ + pThis->pCache->sDir[0] = '\0'; + return; + } + if(strlen(sDir) >= sizeof(pThis->pCache->sDir)){ + daslog_warn_v("Node cache directory name too long, caching is off: %s", sDir); + pThis->pCache->sDir[0] = '\0'; + return; + } + /* sDir may be our buffer, from DasCat_cacheDir(), to change the age */ + if(sDir != pThis->pCache->sDir) strcpy(pThis->pCache->sDir, sDir); + pThis->pCache->nMaxAgeSec = nMaxAgeSec; +} + +const char* DasCat_cacheDir(const DasCat* pThis) +{ + if((pThis->pCache == NULL)||(pThis->pCache->sDir[0] == '\0')) return NULL; + return pThis->pCache->sDir; +} + +int DasCat_cacheAge(const DasCat* pThis) +{ + return (pThis->pCache == NULL) ? 0 : pThis->pCache->nMaxAgeSec; +} + +/* ************************************************************************* */ +/* Root policy */ + +/* Roots are built on first use, so the policy is frozen by then */ +/* das_strdup() gives NULL for "", and these fields promise a string */ +static char* _dascat_strdupz(const char* s) +{ + return ((s == NULL)||(s[0] == '\0')) ? (char*)calloc(1, 1) : das_strdup(s); +} + +static bool _DasCat_frozen(const DasCat* pThis) +{ + if(pThis->uOwned == 0) return false; + daslog_error("Catalog policy can't change after a path has been resolved"); + return true; +} + +DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri) +{ + if(_DasCat_frozen(pThis)) return DASERR_CAT; + if(pThis->bSeed){ + daslog_error_v("Relative paths start at the seed, %s", pThis->sSeedUri[0] ? pThis->sSeedUri : "a root"); + return DASERR_CAT; + } + if((sUri == NULL)||(sUri[0] == '\0')){ + daslog_error("A path prefix can't be empty"); + return DASERR_CAT; + } + if(das_isUrlOrFile(sUri)){ + daslog_error_v("'%s' is a location, a path prefix is a catalog URI", sUri); + return DASERR_CAT; + } + free(pThis->sPrefix); + pThis->sPrefix = _dascat_expandUri(sUri); + /* A prefix is joined to relative paths by plain concatenation, so it + must end in the container's separator. A bare shorthand names the + container; its separator is known, supply it. */ + if((strcmp(pThis->sPrefix, D2URI_SITES) == 0)||(strcmp(pThis->sPrefix, D2URI_ROOT "test") == 0)){ + char* s = das_string("%s:/", pThis->sPrefix); + free(pThis->sPrefix); + pThis->sPrefix = s; + } + pThis->bPrefixGiven = true; + return DAS_OKAY; +} + +const char* DasCat_prefix(const DasCat* pThis) +{ + return pThis->bSeed ? pThis->sSeedUri : pThis->sPrefix; +} + +bool DasCat_hasSeed(const DasCat* pThis){ return pThis->bSeed; } + +/* The mount whose URI covers sUri, or -1. Longest match, as the mount + * table itself resolves, though mounts rarely nest. */ +static int _DasCat_mountFor(const DasCat* pThis, const char* sUri) +{ + int nBest = -1; + size_t uBest = 0; + for(size_t u = 0; u < pThis->uMounts; ++u){ + size_t uLen = strlen(pThis->aMounts[u].sUri); + if((uLen > uBest) && (strncmp(sUri, pThis->aMounts[u].sUri, uLen) == 0)){ + nBest = (int)u; + uBest = uLen; + } + } + return nBest; +} + +/* Add to the mount list, the URI already checked */ +static DasErrCode _DasCat_addMount(DasCat* pThis, const char* sUrl, const char* sUri) +{ + struct das_cat_mount* a = (struct das_cat_mount*)realloc( + pThis->aMounts, (pThis->uMounts + 1) * sizeof(struct das_cat_mount) + ); + if(a == NULL) return das_error(DASERR_CAT, "Couldn't grow the mount list"); + pThis->aMounts = a; + a[pThis->uMounts].sUrl = das_strdup(sUrl); + a[pThis->uMounts].sUri = _dascat_expandUri(sUri); + pThis->uMounts += 1; + return DAS_OKAY; +} + +/* The checks shared by seeds and mounts */ +static bool _dascat_locationOk(const char* sUrl, const char* sUri, bool bUriNeeded) +{ + if(!das_isUrlOrFile(sUrl)){ + daslog_error_v("A node is loaded from a URL or a file, '%s' is neither", sUrl ? sUrl : ""); + return false; + } + if((sUri == NULL)||(sUri[0] == '\0')){ + if(!bUriNeeded) return true; + daslog_error("A mount needs the catalog URI to put the node at"); + return false; + } + if(das_isUrlOrFile(sUri)){ + daslog_error_v("'%s' is a location, a catalog URI was expected", sUri); + return false; + } + return true; +} + +DasErrCode DasCat_setSeed(DasCat* pThis, const char* sUrl, const char* sUri) +{ + if(_DasCat_frozen(pThis)) return DASERR_CAT; + if(pThis->bSeed){ + daslog_error("A session has one seed, and one is already set"); + return DASERR_CAT; + } + if(pThis->bPrefixGiven) + daslog_info_v("Relative paths started at %s, now they start at the seed", pThis->sPrefix); + if(!_dascat_locationOk(sUrl, sUri, false)) return DASERR_CAT; + pThis->sSeedUrl = das_strdup(sUrl); + pThis->sSeedUri = ((sUri != NULL) && (sUri[0] != '\0')) ? _dascat_expandUri(sUri) : (char*)calloc(1, 1); + pThis->bSeed = true; + return DAS_OKAY; +} + +const char* DasCat_seedUrl(const DasCat* pThis){ return pThis->sSeedUrl; } + +DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUri) +{ + if(_DasCat_frozen(pThis)) return DASERR_CAT; + if(!_dascat_locationOk(sUrl, sUri, true)) return DASERR_CAT; + return _DasCat_addMount(pThis, sUrl, sUri); +} + +size_t DasCat_mountCount(const DasCat* pThis){ return pThis->uMounts; } + +/* ************************************************************************* */ +/* Resolution */ + +/* The mount table, built on first use: a synthetic node whose children + * are the mounts by URI, longest URI first so a mount under another + * shadows it. It never falls through to the federated catalog; that is + * decided here, see _DasCat_byUri(). */ +static DasNode* _DasCat_table(DasCat* pThis) +{ + if(pThis->pTable != NULL) return pThis->pTable; + if(pThis->uMounts == 0) return NULL; + + const char** aUrl = (const char**)calloc(pThis->uMounts, sizeof(char*)); + const char** aUri = (const char**)calloc(pThis->uMounts, sizeof(char*)); + if((aUrl == NULL)||(aUri == NULL)){ + das_error(DASERR_CAT, "Couldn't allocate the mount table"); + free((void*)aUrl); free((void*)aUri); /* quiet msvc */ + return NULL; + } + for(size_t u = 0; u < pThis->uMounts; ++u){ + aUrl[u] = pThis->aMounts[u].sUrl; + aUri[u] = pThis->aMounts[u].sUri; + } + /* The table matches children in document order, so a mount at + site:/uiowa listed before one at site:/uiowa/juno/wav would shadow + it. Longest URI first makes the order mounts were given in + irrelevant, as it should be: lists in a catalog are never + preferences. Insertion sort, there are a handful at most. */ + for(size_t i = 1; i < pThis->uMounts; ++i){ + for(size_t j = i; (j > 0) && (strlen(aUri[j]) > strlen(aUri[j-1])); --j){ + const char* s = aUri[j]; aUri[j] = aUri[j-1]; aUri[j-1] = s; + s = aUrl[j]; aUrl[j] = aUrl[j-1]; aUrl[j-1] = s; + } + } + pThis->pTable = _dasnode_newMountTable(aUrl, aUri, pThis->uMounts); + free((void*)aUrl); /* quiet msvc */ + free((void*)aUri); /* quiet msvc */ + return _DasCat_keep(pThis, pThis->pTable); +} + +/* A full catalog URI under the current policy, sLastUri already set. + * Mounts are opportunistic: a URI one covers is walked from the mount + * table, anything else from the seed, or from the compiled in seed when + * none is set. Shared by DasCat_resolve() and DasCat_nearest(), which + * has a URI in hand and must not go through the location test. */ +static DasNode* _DasCat_byUri(DasCat* pThis, const char* sUri) +{ + if(_DasCat_mountFor(pThis, sUri) >= 0){ + DasNode* pTable = _DasCat_table(pThis); + if(pTable == NULL) return NULL; + return _DasCat_walkFrom(pThis, pTable, sUri); + } + if(pThis->bSeed){ + if(!DasCat_covers(pThis, sUri)){ + daslog_debug_v("%s is outside the seed %s", sUri, pThis->sSeedUri[0] ? pThis->sSeedUri : "(a root)"); + return NULL; + } + DasNode* pSeed = _DasCat_seedNode(pThis); + if(pSeed == NULL) return NULL; + /* The seed itself is asked for by its prefix; it may be a source, + which no walk can start from, so it is handed back directly */ + const char* sRest = sUri + strlen(pThis->sSeedUri); + if(sRest[0] == '\0') return pSeed; + return _DasCat_walkFrom(pThis, pSeed, sRest); + } + DasNode* pDas2 = _DasCat_das2(pThis); + if(pDas2 == NULL) return NULL; + return _DasCat_walkFrom(pThis, pDas2, sUri); +} + +DasNode* DasCat_resolve(DasCat* pThis, const char* sPath) +{ + bool bNoPath = (sPath == NULL)||(sPath[0] == '\0'); + _dascat_setStr(&pThis->sLastPath, bNoPath ? NULL : sPath); + free(pThis->sLastUri); pThis->sLastUri = NULL; + free(pThis->sNear); pThis->sNear = NULL; + free(pThis->sNearRest); pThis->sNearRest = NULL; + + /* A node by location: a root, the location stands in for a + name in messages */ + if(!bNoPath && das_isUrlOrFile(sPath)){ + pThis->sLastUri = das_strdup(sPath); + return _DasCat_keep(pThis, _dascat_loadUrl( + sPath, NULL, pThis->pMgr, pThis->sAgent, pThis->pCache + )); + } + + /* With a seed, relative paths start at it; a full URI goes through the + * mounts or nowhere */ + if(pThis->bSeed){ + if(bNoPath){ + pThis->sLastUri = _dascat_strdupz(pThis->sSeedUri); + return _DasCat_seedNode(pThis); + } + if(das_isUri(sPath)){ + pThis->sLastUri = _dascat_expandUri(sPath); + return _DasCat_byUri(pThis, pThis->sLastUri); + } + DasNode* pSeed = _DasCat_seedNode(pThis); + if(pSeed == NULL){ + pThis->sLastUri = _dascat_strdupz(pThis->sSeedUri); + return NULL; + } + /* Names below a root are the path from it; below a prefix the + seed's separator joins them. The full name then goes through + the same door as a URI, so a mount under the seed is consulted + whichever way the path was spelled. */ + if(pThis->sSeedUri[0] == '\0'){ + pThis->sLastUri = das_strdup(sPath); + } + else{ + const char* sSep = DasNode_isCatalog(pSeed) ? DasNode_pathSep(pSeed) : "/"; + pThis->sLastUri = das_string("%s%s%s", pThis->sSeedUri, sSep ? sSep : "/", sPath); + } + return _DasCat_byUri(pThis, pThis->sLastUri); + } + + /* Otherwise relative paths start at the prefix, mounts or not */ + if(bNoPath){ + /* The prefix with its trailing separator removed names the container */ + pThis->sLastUri = das_strdup(pThis->sPrefix); + size_t u = strlen(pThis->sLastUri); + while((u > 0) && ((pThis->sLastUri[u-1] == '/')||(pThis->sLastUri[u-1] == ':'))) + pThis->sLastUri[--u] = '\0'; + } + else if(das_isUri(sPath)) + pThis->sLastUri = _dascat_expandUri(sPath); + else + pThis->sLastUri = das_string("%s%s", pThis->sPrefix, sPath); + + return _DasCat_byUri(pThis, pThis->sLastUri); +} + +const char* DasCat_lastUri(const DasCat* pThis){ return pThis->sLastUri; } + +bool DasCat_covers(const DasCat* pThis, const char* sUri) +{ + /* Mounts fall through to the federated catalog; only a seed walls paths + * off. Names below a root seed are paths from it, so anything with a + * scheme in front is outside it. */ + if(!pThis->bSeed) return true; + if(_DasCat_mountFor(pThis, sUri) >= 0) return true; + if(pThis->sSeedUri[0] == '\0') return !das_isUri(sUri); + return strncmp(sUri, pThis->sSeedUri, strlen(pThis->sSeedUri)) == 0; +} + +DasNode* DasCat_nearest(DasCat* pThis, const char** psRest) +{ + if(psRest != NULL) *psRest = NULL; + if(pThis->pNearNode != NULL){ + /* A root: nothing to resolve again, and no URI to report, so the + last path and URI say so with an empty string */ + DasNode* pNode = pThis->pNearNode; + pThis->pNearNode = NULL; + _dascat_setStr(&pThis->sLastPath, NULL); + free(pThis->sLastUri); + pThis->sLastUri = (char*)calloc(1, 1); + if(psRest != NULL) *psRest = pThis->sNearRest; + return pNode; + } + if(pThis->sNear == NULL) return NULL; + + /* The walk that failed said how far it got. Loading that container + * again is cheap, loaded children or the node cache, and it has the + * documented effect on the last path and URI: afterwards they name the + * nearest node. The walk's memory is taken first, the resolve clears + * it. */ + char* sNear = pThis->sNear; pThis->sNear = NULL; + char* sRest = pThis->sNearRest; pThis->sNearRest = NULL; + char* sOrigPath = pThis->sLastPath; pThis->sLastPath = das_strdup(sNear); + char* sOrigUri = pThis->sLastUri; pThis->sLastUri = sNear; + + DasNode* pNode = _DasCat_byUri(pThis, pThis->sLastUri); + if(pNode == NULL){ + /* It loaded a moment ago, so a server went away between calls; + * keep the caller's view of the original failure */ + free(sRest); + free(pThis->sLastPath); pThis->sLastPath = sOrigPath; + free(pThis->sLastUri); pThis->sLastUri = sOrigUri; + return NULL; + } + free(sOrigPath); + free(sOrigUri); + free(pThis->sNearRest); + pThis->sNearRest = sRest; /* the caller's pointer, valid until the next resolve */ + if(psRest != NULL) *psRest = sRest; + return pNode; +} + +DasNode* DasCat_seed(DasCat* pThis) +{ + return pThis->bSeed ? _DasCat_seedNode(pThis) : _DasCat_das2(pThis); +} + +DasNode* DasCat_subNode(DasCat* pThis, DasNode* pNode, const char* sRelPath) +{ + if(pNode == NULL){ + daslog_error("No node to load a child of"); + return NULL; + } + return _DasCat_walkFrom(pThis, pNode, sRelPath); +} + +/* ************************************************************************* */ +/* Sources */ + +DasSrc* DasCat_newSrc(DasCat* pThis, const char* sPath) +{ + DasNode* pNode = DasCat_resolve(pThis, sPath); + return (pNode == NULL) ? NULL : new_DasSrc_node(pNode); +} diff --git a/das3/catalog.h b/das3/catalog.h new file mode 100644 index 0000000..e6d326e --- /dev/null +++ b/das3/catalog.h @@ -0,0 +1,436 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/** @file catalog.h A catalog session: where paths start, credentials and + * node memory in one object, with mounts */ + +#ifndef _das_catalog_h_ +#define _das_catalog_h_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @defgroup catalog Catalogs + * Finding data sources: the federated das catalog, catalog URIs, and the + * objects that walk it. + * + * A das catalog is a tree of small JSON documents, mostly on web servers, + * leading from a well known root down to data source definitions. Each + * document is a node. Container nodes (Catalog, Collection, SourceSet) + * list their children by name along with the URLs each child can be + * loaded from. Leaf nodes (HttpStreamSrc, FileTimeAgg) describe how to + * get data. The tree is federated: the root at das2.org names a few top + * level trees, and any node may have children maintained by a different + * organization. A path is resolved one network request per level, and + * only the levels on the way to the node asked for are ever loaded. + * + * @par Names + * + * A node's URI is composed by walking down from the root: each container + * appends its separator and the child's name to its URI. A node does + * not know its name, the walk gives it one, and the root, which has + * nothing above it, has none. The result looks like a file path with a + * URI scheme in front: + * + * tag:das2.org,2012:site:/uiowa/juno/wav/survey + * + * The RFC 4151 "tag" scheme is the usual one, though others such as "doi" + * work just as well. URLs are not used as names because they change + * whenever an organization reshuffles its servers, while a name must not. + * For that reason a name may not start with a locator scheme, "http", + * "https", "ftp" and so on; the library reads those as locations, see + * das_isUrlOrFile(). + * + * In daily use the prefix "tag:das2.org,2012:site:/" is noise, so the + * library also accepts the relative form "uiowa/juno/wav/survey" and the + * shorthand "site:/uiowa/juno/wav/survey". + * + * Since names come from the walk, a local file can be declared to be any + * node. For testing, the file + * + * ./survey.json + * + * can be mounted as + * + * tag:das2.org,2012:site:/uiowa/juno/wav/survey + * + * and walking proceeds from there as if the file had been fetched from + * the catalog, see DasCat_mount() and DasCat_setSeed(). A file loaded + * with no name at all is a root and has no URI. + * + * @par Reading a node + * + * Open a catalog session, resolve a path and read the node: + * + * @code + DasCat* pCat = new_DasCat(NULL, "my_program/1.0"); + + DasNode* pNode = DasCat_resolve(pCat, "uiowa/juno/wav/survey"); + if((pNode == NULL) || !DasNode_isStreamSrc(pNode)) handle_error(); + + // The query interface of the source, laid out by the das catalog schema + const DasJdo* pProto = DasNode_getJdo(pNode, "protocol"); + ... + del_DasCat(pCat); // frees every node the session loaded + @endcode + * + * Programs that want to query a data source rather than inspect its node + * use DasCat_newSrc(), which resolves the path and builds a DasSrc, see + * source.h. + * + * Two objects do the work. DasCat, in das3/catalog.h, is a session: it + * holds where relative paths start, see DasCat_setPrefix() and + * DasCat_setSeed(), any mounts, the credentials and user agent every load + * uses, and the memory of every node it loads. DasNode, in das3/node.h, + * is one node: its type, URI, source URL and parsed document, with + * DasNodeWalk to step down to a child. Most programs only ever construct + * a DasCat. The node interface is for programs that fetch for themselves, + * see "Nodes without a session" in node.h. + * + * @par Seeds, mounts, and the root + * + * A seed is a node a program starts from with nothing above it in memory. + * The compiled in seed is the index of the das catalog, see + * das_seed_urls(); a session loads it on first use and keeps it, so the + * tree below it grows as paths are resolved. A file can be a seed too, + * see DasCat_setSeed(), given a prefix that names it, or none. A seed + * given no prefix is a root: the one node with no URI, from which names + * are composed starting from nothing. A mount, see DasCat_mount(), is a + * node grafted at a name in the session's name space, whichever seed + * defines it: a path from a root, a URI under a prefix, or a federated + * URI when no seed is set. It is consulted before the seed for anything + * under that name, however the path was spelled. + */ + +/** @addtogroup catalog + * @{ + */ + +/** The catalog URI prefix relative paths start from unless told otherwise */ +#define DASCAT_DEF_PREFIX "tag:das2.org,2012:site:/" + +/** How long, in seconds, a new session trusts a cached catalog node */ +#define DASCAT_DEF_CACHE_SEC 3600 + +/** A catalog session, opaque. + * + * A DasCat turns catalog paths into nodes. It remembers where relative + * paths start, which credentials and user agent to send, and every node it + * has loaded, so a program, a MATLAB extension or a Python module can say + * "resolve this path" and get a node back without managing seeds itself. + * + * Paths given to DasCat_resolve() are catalog paths, not web paths, in any + * of these forms: + * + * - relative to the prefix: "uiowa/juno/wav/survey" + * - the shorthands "site:/uiowa/..." and "test:/..." + * - a full catalog URI: "tag:das2.org,2012:site:/uiowa/..." + * - a node by location, a URL or a file: "https://host/node.json", + * "./node.json". Such a node is a root, with no URI; + * DasCat_lastUri() reports the location in its place. + * + * Where relative paths start is one setting with two ways to set it: + * + * - By default they start at DASCAT_DEF_PREFIX in the federated + * catalog. DasCat_setPrefix() moves that. + * - DasCat_setSeed() makes a node loaded by location the seed: relative + * paths start at it, full URIs resolve only under its prefix or under + * a mount, and nothing else can be reached. This is how an offline + * catalog is assembled from a few files. + * + * DasCat_mount() puts a node, loaded by location, at a catalog URI over + * the federated catalog. Paths under the URI resolve through the mounted + * node, everything else through the compiled in seed as usual. Mounts + * never move where relative paths start. + * + * Policy is set before the first resolve. Nodes returned by + * DasCat_resolve() are owned by the catalog and live until del_DasCat(). + * + * @code + DasCat* pCat = new_DasCat(NULL, "my_program"); + DasNode* pNode = DasCat_resolve(pCat, "uiowa/juno/wav/survey"); + if(pNode == NULL){ + const char* sRest = NULL; + DasNode* pNear = DasCat_nearest(pCat, &sRest); + // pNear is the deepest catalog that did load, sRest what it lacks + } + del_DasCat(pCat); + @endcode + */ +typedef struct das_cat DasCat; + + +/** Create a catalog session + * + * The returned session has its node cache enabled right out of the gate, + * to turn it off see DasCat_setCache() + * + * @param pMgr Credentials manager consulted when a node or data source asks + * for a login. May be NULL for public access only. Ownership + * passes to the catalog, del_DasCat() frees it. + * @param sAgent User agent string sent with every request, copied. NULL + * sends the library default. + * @returns a new catalog, free with del_DasCat(). + * @memberof DasCat + */ +DAS_API DasCat* new_DasCat(DasCredMngr* pMgr, const char* sAgent); + +/** Free a catalog session, every node it loaded, and its credentials manager + * @memberof DasCat + */ +DAS_API void del_DasCat(DasCat* pThis); + +/** The credentials manager, borrowed, for data requests made by the caller + * @returns the manager given to new_DasCat(), or NULL + * @memberof DasCat + */ +DAS_API DasCredMngr* DasCat_credMngr(const DasCat* pThis); + +/** The user agent string, borrowed + * @returns the string given to new_DasCat(), or NULL + * @memberof DasCat + */ +DAS_API const char* DasCat_agent(const DasCat* pThis); + +/** Change where, and for how long, this session keeps copies of catalog + * nodes fetched over HTTP + * + * A new session already caches: copies go under das_userdir() in + * cache/nodes and are trusted for DASCAT_DEF_CACHE_SEC. Walking a catalog + * costs one request per level, so this is what makes the second run of a + * program fast. A copy older than that is not thrown away: the server is + * asked whether it changed, using the ETag or Last-Modified it came with, + * and only a changed node is downloaded again. A server that cannot be + * reached leaves the old copy in use, with a warning. Nodes read from + * local files are not cached, and neither are nodes built from bytes a + * program fetched itself, see new_DasNode_json(). + * + * @param sDir Directory for the copies, created on first use. NULL or an + * empty string turns caching off for this session. Passing + * DasCat_cacheDir() is allowed, and can be used to change the + * default node expiration time. + * @param nMaxAgeSec Copies older than this many seconds are checked with + * the server before use. Zero or less means check every time, + * which costs one round trip per level and no body bytes when the + * server supports validators, and a full fetch when it does not. + * @memberof DasCat + */ +DAS_API void DasCat_setCache(DasCat* pThis, const char* sDir, int nMaxAgeSec); + +/** The directory this session caches nodes in, or NULL if caching is off + * @memberof DasCat + */ +DAS_API const char* DasCat_cacheDir(const DasCat* pThis); + +/** How many seconds this session trusts a cached node, see DasCat_setCache() + * @memberof DasCat + */ +DAS_API int DasCat_cacheAge(const DasCat* pThis); + +/** Where relative paths start in the federated catalog + * + * The shorthands "site:" and "test:" are expanded; given alone they name + * the container and its ":/" child separator is supplied, so "site:" is + * the same prefix as "tag:das2.org,2012:site:/". A URL or file is + * refused; a node to start from is a seed, see DasCat_setSeed(). + * + * @param sUri A catalog URI prefix, typically ending in the separator the + * container uses, such as "tag:das2.org,2012:test:/" + * @returns DAS_OKAY, or DASERR_CAT with the reason logged: a location, + * an empty string, or a seed already set, since relative paths + * then start at the seed + * @memberof DasCat + */ +DAS_API DasErrCode DasCat_setPrefix(DasCat* pThis, const char* sUri); + +/** Where relative paths start: the seed's URI when one is set, else the + * prefix, DASCAT_DEF_PREFIX unless DasCat_setPrefix() was called + * @memberof DasCat + */ +DAS_API const char* DasCat_prefix(const DasCat* pThis); + +/** Make a node loaded by location the seed, and the mounts the whole catalog + * + * Relative paths then start at this node, full URIs resolve only under + * its prefix or under a mount, and everything else fails, which + * DasCat_covers() can tell apart from an ordinary miss. Further DasCat_mount() calls, before + * or after, add to the offline catalog. A session has one seed, and one + * place relative paths start: a prefix set earlier is superseded, and + * DasCat_setPrefix() is refused afterwards. + * + * The prefix declares what the node is: a development server's copy of a + * catalog branch stands for the branch's federated name, so requisitions + * made against it are good against the live catalog too. Without a + * prefix the seed is a root: it has no URI, nodes below it are named by + * their path from it, and requisitions made against sources below it + * carry those names. Give the real URI when the file stands in for a + * catalog node, so that requisitions match it. + * + * @param sUrl A URL or file path of a node document + * @param sUri The catalog URI the node has, shorthands accepted, or NULL + * for a root + * @returns DAS_OKAY, or DASERR_CAT with the reason logged + * @memberof DasCat + */ +DAS_API DasErrCode DasCat_setSeed(DasCat* pThis, const char* sUrl, const char* sUri); + +/** The location given to DasCat_setSeed(), or NULL if none was + * @memberof DasCat + */ +DAS_API const char* DasCat_seedUrl(const DasCat* pThis); + +/** The seed node, loading it if need be + * + * The node given to DasCat_setSeed(), or the index of the das catalog, + * a root with no URI whose children are the top level catalogs. Owned + * by the session. For programs that list the catalog from the top or + * walk it themselves with DasCat_subNode(). + * + * @returns the node, or NULL after logging why not + * @memberof DasCat + */ +DAS_API DasNode* DasCat_seed(DasCat* pThis); + +/** Put a node, loaded by location, at a name over the catalog + * + * Paths under sUri load from sUrl and shadow whatever the catalog says + * for them, everything else resolves through the seed as usual. The + * name is in the session's name space: a federated URI, or with a seed + * set, see DasCat_setSeed(), a URI under its prefix or a path from it + * when it is a root, and the mount then joins the offline catalog. May + * be called more than once; when one mount point is under another the + * longer name wins. A mount does not move where relative paths start. + * + * @param sUrl Where to load the node from, a URL or a file + * @param sUri The name to mount it at, shorthands accepted + * @returns DAS_OKAY, or DASERR_CAT with the reason logged + * @memberof DasCat + */ +DAS_API DasErrCode DasCat_mount(DasCat* pThis, const char* sUrl, const char* sUri); + +/** Number of nodes mounted with DasCat_mount(); the seed is not one + * @memberof DasCat + */ +DAS_API size_t DasCat_mountCount(const DasCat* pThis); + +/** True if a seed is set, so paths reach only it and the mounted nodes + * @memberof DasCat + */ +DAS_API bool DasCat_hasSeed(const DasCat* pThis); + +/** Resolve a path to a node + * + * Network access happens here, one request per catalog level not yet + * loaded. Nodes are kept, a second resolve of the same path costs nothing. + * + * @param sPath A catalog path in any of the forms listed for DasCat. NULL + * or empty means where relative paths start: the seed, or the + * container the prefix names. + * @returns the node, owned by the catalog, or NULL if the path did not + * resolve. DasCat_lastUri() then gives the URI that was tried and + * DasCat_nearest() the deepest node that did load. + * @memberof DasCat + */ +DAS_API DasNode* DasCat_resolve(DasCat* pThis, const char* sPath); + +/** The full catalog URI of the last path given to DasCat_resolve(), as the + * policy understood it, resolved or not. For a path by location this is + * the location, and for a root, which has no URI, it is "". + * @returns the URI, or NULL before the first resolve + * @memberof DasCat + */ +DAS_API const char* DasCat_lastUri(const DasCat* pThis); + +/** Can this URI be reached under the current policy? + * + * False when a seed is set and sUri is neither under its prefix nor + * under a mount. Names below a root seed are paths from it, so under + * one anything with a scheme in front is outside. A failed resolve of + * a URI this returns false for is a policy miss, not a catalog miss, and + * the message to the user should say so. + * @memberof DasCat + */ +DAS_API bool DasCat_covers(const DasCat* pThis, const char* sUri); + +/** After a failed resolve, the deepest catalog node on the path that did load + * + * The walk that failed remembers the deepest container it searched and + * the part of the path below it. This loads that container again, which + * costs nothing when it is still in memory and one cache read otherwise. + * DasCat_lastUri() afterwards names the node returned, so a caller can + * say "no child X under URI Y" and list what Y does have with + * DasNode_subName(). + * + * @param psRest If not NULL, receives a pointer to the part of the failed + * path below the node returned, such as "survey/das2", valid until + * the next resolve. NULL when nothing is returned. + * @returns the node, owned by the catalog, or NULL if the last resolve + * succeeded, the path was a location, or no container on the path + * loaded at all, as when no mirror of the das catalog index answered. + * The node may be a root, the seed itself, and DasCat_lastUri() + * is then "". + * @memberof DasCat + */ +DAS_API DasNode* DasCat_nearest(DasCat* pThis, const char** psRest); + +/** Load a child of a node, with this session's settings + * + * For a program that has resolved a container, listed its children, see + * DasNode_subName() and dassrc_loadable(), and picked one. The child is + * loaded with the session's credentials, agent and cache, attached under + * pNode, and lives as long as the tree it is in. The same as resolving + * the child's full URI, without composing it. + * + * @param pNode A container node, usually one this session resolved. A + * tree the program loaded itself works too and keeps the child. + * @param sRelPath The path below pNode, "das3" or "survey/das2", with or + * without the separator pNode uses in front. Empty means pNode. + * @returns the node, or NULL after logging why not. DasCat_nearest() + * then says how far the walk got. + * @memberof DasCat + */ +DAS_API DasNode* DasCat_subNode(DasCat* pThis, DasNode* pNode, const char* sRelPath); + +/** Resolve a path and build the data source it leads to. + * + * Resolves the path as DasCat_resolve() does and hands the node to + * new_DasSrc_node(). The path has to end at a data source not a + * Container, and it must be a catalog node type that das2C understands. + * + * @see dassrc_loadable() + * + * @param sPath A catalog path in any form DasCat_resolve() accepts, + * including a URL or file name. + * @returns A new source, to be freed with del_DasSrc() and independent + * of this session afterwards, or NULL after logging why not. As + * with a failed resolve, DasCat_lastUri() and DasCat_nearest() + * say how far the path got. + * @memberof DasCat + */ +DAS_API DasSrc* DasCat_newSrc(DasCat* pThis, const char* sPath); + +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* _das_catalog_h_ */ diff --git a/das3/core.h b/das3/core.h index 737bb21..ec4d582 100644 --- a/das3/core.h +++ b/das3/core.h @@ -232,7 +232,9 @@ das3/form_*.h Mathematical formalisms on composite types das3/utf8.h Counting characters instead of bytes das3/toml.h Reading TOML configuration files das3/dft.h Properly scaled power spectral densities -das3/node.h das catalog nodes, includes credentials.h and json.h +das3/catalog.h Finding data sources in the federated catalog system +das3/source.h Data source descriptions, their properties and controls +das3/req.h Requisitions, what a client wants from a data source das3/http.h Reading streams from servers, credentials.h handles logins das3/builder.h Collect a whole das2 or das3 stream into DasDs objects das3/packet.h das2 packet descriptors, new code should use DasDs instead diff --git a/das3/defs.h b/das3/defs.h index da03cc3..bfc5a69 100644 --- a/das3/defs.h +++ b/das3/defs.h @@ -269,7 +269,9 @@ typedef int DasErrCode; #define DASERR_URI 42 #define DASERR_FORM 43 #define DASERR_GEN 44 -#define DASERR_MAX 44 +#define DASERR_CAT 45 +#define DASERR_SRC 46 +#define DASERR_MAX 46 #ifdef __cplusplus } diff --git a/das3/dsdf.h b/das3/dsdf.h index de65510..586e170 100644 --- a/das3/dsdf.h +++ b/das3/dsdf.h @@ -27,9 +27,6 @@ extern "C" { #endif -/** @addtogroup catalog - * @{ - */ /** Parse a DSDF file into a descriptor object. @@ -101,7 +98,6 @@ DAS_API char* dsdf_valToNormParam( const char* sRawParam, char* sNormParam, size_t uNormLen ); -/** @} */ #ifdef __cplusplus } diff --git a/das3/http.c b/das3/http.c index a251713..3f55501 100644 --- a/das3/http.c +++ b/das3/http.c @@ -1,6 +1,6 @@ -/* Copyright (C) 2017-2019 Chris Piker +/* Copyright (C) 2017-2026 Chris Piker * - * This file is part of libdas2, the Core Das2 C Library. + * This file is part of das2C, the Core Das2 C Library. * * Libdas2 is free software; you can redistribute it and/or modify it under * the terms of the GNU Lesser General Public License version 2.1 as published @@ -12,7 +12,7 @@ * more details. * * You should have received a copy of the GNU Lesser General Public License - * version 2.1 along with libdas2; if not, see . + * version 2.1 along with das2C; if not, see . */ #define _POSIX_C_SOURCE 200112L @@ -53,6 +53,12 @@ #include "info_priv.h" #include "util_priv.h" +#ifdef _WIN32 +#define strncasecmp _strnicmp +#else +#include +#endif + #ifdef _WIN32 typedef ptrdiff_t ssize_t; #endif @@ -81,6 +87,7 @@ char html5[256] = { #define LIBDAS_USER_AGENT "libdas3" #define HTTP_OK 200 +#define HTTP_NotModified 304 #define HTTP_MovedPerm 301 #define HTTP_Found 302 #define HTTP_TempRedir 307 /* Treat as 307 */ @@ -91,8 +98,6 @@ char html5[256] = { #define HTTP_NotFound 404 #define HTTP_Error 500 -#define _QDEF(x) #x -#define QDEF(x) _QDEF(x) /* ************************************************************************* */ /* The global SSL context and mutexes for manipulating it. We use lazy @@ -272,6 +277,29 @@ char* das_ssl_getErr(const void* vpSsl, int nRet) /* ************************************************************************* */ /* Parsing a URL */ +/* Copy input into a fixed URL field up to the first stop character or the + * end of input. A part that does not fit is an error, never a truncation: + * a shortened path or query would reach the server as a different request, + * and a shortened dataset would match the wrong saved password. Returns + * where the copy stopped, or NULL with pRes->sError set. */ +static const char* _das_url_field( + DasHttpResp* pRes, char* sOut, size_t uMax, const char* pIn, + const char* sStop, const char* sWhat, const char* sUrl +){ + size_t u = 0; + while((*pIn != '\0') && (strchr(sStop, *pIn) == NULL)){ + if(u == uMax){ + pRes->sError = das_string( + "The %s in URL %s is longer than %zu bytes", sWhat, sUrl, uMax + ); + return NULL; + } + sOut[u++] = *pIn++; + } + sOut[u] = '\0'; + return pIn; +} + bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl) { struct das_url* pUrl = &(pRes->url); @@ -279,79 +307,53 @@ bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl) pRes->nSockFd = -1; pRes->nCode = -1; - pRes->url.sPort[0] = '8'; pRes->url.sPort[1] = '0'; - /* Get the scheme, this is a PITA but I don't want a large library - * dependency and uriparser doesn't want to deal with utf-8 natively. - * I'm sure curl would take care of it nicely, but does that exist on - * windows? */ - char* pOut = pUrl->sScheme; - const char* pIn = sUrl; - while((*pIn != '\0')&&(*pIn != ':')&&((pOut - pUrl->sScheme) <= DASURL_SZ_SCHEME )){ - *pOut = *pIn; ++pOut; ++pIn; - } - if((strcmp("http", pUrl->sScheme)!=0 )&&(strcmp("https", pUrl->sScheme)!=0)){ + /* Hand rolled on purpose: uriparser does not take UTF-8 natively and + * curl is a large dependency to carry for a handful of fields. */ + const char* pIn = _das_url_field(pRes, pUrl->sScheme, DASURL_SZ_SCHEME, sUrl, ":", "scheme", sUrl); + if(pIn == NULL) return false; + if((strcmp("http", pUrl->sScheme) != 0)&&(strcmp("https", pUrl->sScheme) != 0)){ pRes->sError = das_string("Unknown scheme, %s", pUrl->sScheme); return false; } - if(strcmp("https", pUrl->sScheme) == 0){ - pUrl->sPort[0] = '4'; pUrl->sPort[1] = '4'; pUrl->sPort[2] = '3'; - } - else{ - pUrl->sPort[0] = '8'; pUrl->sPort[1] = '0'; - } + strcpy(pUrl->sPort, (strcmp("https", pUrl->sScheme) == 0) ? "443" : "80"); while(*pIn == ':') pIn++; while(*pIn == '/') pIn++; /* Advance past a couple /'s or four */ - pOut = pUrl->sHost; - while( (*pIn != '\0')&&(*pIn != ':')&&(*pIn != '/')&&(*pIn != '?')&& - ((pOut - pUrl->sHost) <= DASURL_SZ_HOST)){ - *pOut = *pIn; ++pOut; ++pIn; - } - if((strlen(pUrl->sHost) < 1)||((pOut - pUrl->sHost) == (DASURL_SZ_HOST))){ + pIn = _das_url_field(pRes, pUrl->sHost, DASURL_SZ_HOST, pIn, ":/?", "host", sUrl); + if(pIn == NULL) return false; + if(pUrl->sHost[0] == '\0'){ pRes->sError = das_string("Invalid host in URL %s", sUrl); return false; } - char sPort[32] = {'\0'}; - pOut = sPort; - int nPort; if(*pIn == ':'){ - while( (*pIn != '\0')&&(*pIn != '/')&&((pOut - sPort) < 32) ){ - *pOut = *pIn; ++pOut; ++pIn; - } - nPort = 0; - if((strlen(sPort) < 1) || ((nPort = atoi(sPort)) == 0) || - (nPort > 65536) ){ + /* The colon is a delimiter, not part of the port number */ + char sPort[DASURL_SZ_PORT + 1]; + pIn = _das_url_field(pRes, sPort, DASURL_SZ_PORT, pIn + 1, "/?", "port", sUrl); + if(pIn == NULL) return false; + int nPort = atoi(sPort); + if((nPort < 1)||(nPort > 65535)){ pRes->sError = das_string("Invalid port in URL %s", sUrl); return false; } - strncpy(pUrl->sPort, sPort, 7); + strcpy(pUrl->sPort, sPort); } - /* don't skip the / it's part of the path */ - - /* Skipping fragments for now */ - pOut = pUrl->sPath; /* Note this can be zero */ - while( (*pIn != '\0')&&(*pIn != '?')&&((pOut - pUrl->sPath)sPath, DASURL_SZ_PATH, pIn, "?", "path", sUrl); + if(pIn == NULL) return false; if(*pIn == '?') ++pIn; - pOut = pUrl->sQuery; - while((*pIn != '\0')&&((pOut - pUrl->sQuery)sQuery, "dataset=")) != NULL){ + pIn = _das_url_field(pRes, pUrl->sDataset, DASURL_SZ_DATASET, pIn + 8, "&", "dataset", sUrl); + if(pIn == NULL) return false; } return true; } @@ -729,7 +731,8 @@ static bool _das_http_connect(DasHttpResp* pRes, struct timeval* pTimeOut) static bool _das_http_getRequest( - DasHttpResp* pRes, const char* sAgent, const char* sAuth, DasBuf* pBuf + DasHttpResp* pRes, const char* sAgent, const char* sAuth, const char* sReqHdrs, + DasBuf* pBuf ){ /* Have to handle the fact that blocking SIGPIPE is different on Apple */ #if defined(__APPLE__) || defined(_WIN32) || defined(__sun) @@ -754,6 +757,8 @@ static bool _das_http_getRequest( /* fprintf(stderr, "Authorization: Basic %s\r\n", sAuth); */ DasBuf_printf(pBuf, "Authorization: Basic %s\r\n", sAuth); } + if((sReqHdrs != NULL)&&(sReqHdrs[0] != '\0')) + DasBuf_printf(pBuf, "%s", sReqHdrs); /* caller's lines, each CRLF ended */ DasBuf_printf(pBuf, "Connection: close\r\n\r\n"); size_t uSent = 0; @@ -839,6 +844,38 @@ static bool _das_http_hdrSearch(DasBuf* pBuf, const char* sField, char* sBuf, si return false; } +bool DasHttpResp_header( + const DasHttpResp* pRes, const char* sField, char* sBuf, size_t uLen +){ + if(uLen > 0) sBuf[0] = '\0'; + if((pRes == NULL)||(pRes->sHeaders == NULL)||(sField == NULL)||(uLen == 0)) + return false; + + /* Field names are case insensitive (RFC 9110), and servers do differ: + Apache sends ETag, GitHub sends etag. Values are trimmed. */ + size_t uField = strlen(sField); + const char* pLine = pRes->sHeaders; + while(*pLine != '\0'){ + const char* pEnd = strstr(pLine, "\r\n"); + size_t uLine = pEnd ? (size_t)(pEnd - pLine) : strlen(pLine); + if((uLine > uField) && (pLine[uField] == ':') && + (strncasecmp(pLine, sField, uField) == 0)){ + const char* pVal = pLine + uField + 1; + const char* pValEnd = pLine + uLine; + while((pVal < pValEnd) && isspace((unsigned char)*pVal)) ++pVal; + while((pValEnd > pVal) && isspace((unsigned char)pValEnd[-1])) --pValEnd; + size_t uVal = (size_t)(pValEnd - pVal); + if(uVal >= uLen) return false; /* does not fit, never truncated */ + memcpy(sBuf, pVal, uVal); + sBuf[uVal] = '\0'; + return true; + } + if(pEnd == NULL) break; + pLine = pEnd + 2; + } + return false; +} + static bool _das_http_setFileName(DasBuf* pBuf, DasHttpResp* pRes) { char sDis[196] = {'\0'}; @@ -1020,25 +1057,34 @@ static bool _das_http_redirect(DasHttpResp* pRes, DasBuf* pBuf) } void DasHttpResp_freeFields(DasHttpResp* pRes){ - if(pRes->pMime) free(pRes->pMime); - if(pRes->sHeaders) free(pRes->sHeaders); + /* Every string here is heap allocated when set, see the das_string() + calls above and _das_http_setFileName(); a failed request leaves + sError behind and nothing else */ + free(pRes->pMime); pRes->pMime = NULL; + free(pRes->sHeaders); pRes->sHeaders = NULL; + free(pRes->sError); pRes->sError = NULL; + free(pRes->sFilename); pRes->sFilename = NULL; } /* ************************************************************************* */ /* Get a message body socket, involves quite a few steps */ +static void _das_http_closeBody(DasHttpResp* pRes); + bool das_http_getBody( const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, float rConSec +){ + return das_http_getBody_ex(sUrl, sAgent, pMgr, pRes, rConSec, NULL); +} + +bool das_http_getBody_ex( + const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, + float rConSec, const char* sReqHdrs ){ DasHttpResp_clear(pRes); /* Sets nSockFd to -1 */ - if(strlen(sUrl) > DASURL_SZ_QUERY){ - pRes->sError = das_string("URL is greater than " QDEF(DASURL_SZ_QUERY) " bytes"); - return false; - } - struct timeval tv = {0, 0}; if(rConSec > 0.0){ tv.tv_sec = (long)rConSec; @@ -1061,7 +1107,7 @@ bool das_http_getBody( } /* Send the request */ - if(! _das_http_getRequest(pRes, sAgent, sAuth, pBuf) ){ + if(! _das_http_getRequest(pRes, sAgent, sAuth, sReqHdrs, pBuf) ){ goto CLEANUP_ERROR; } @@ -1088,6 +1134,16 @@ bool das_http_getBody( del_DasBuf(pBuf); return true; break; + case HTTP_NotModified: + /* Only reachable when sReqHdrs carried a validator. There is no + body, so the transport is released here and the caller has the + headers and nCode to go on. */ + pRes->sHeaders = (char*)calloc(DasBuf_written(pBuf) + 1, sizeof(char)); + DasBuf_read(pBuf, pRes->sHeaders, DasBuf_written(pBuf)); + del_DasBuf(pBuf); + _das_http_closeBody(pRes); + return true; + break; case HTTP_MovedPerm: case HTTP_Found: case HTTP_TempRedir: @@ -1202,84 +1258,146 @@ bool das_http_getBody( /* ************************************************************************* */ -/* A just-give-me-a-bag-of-bytes convenience function */ +/* Body readers: one chunk at a time, shared by the slurp and copy functions */ #define D2CHAR_CHUNK_SZ 16384 +/* Returns bytes read, 0 at end of body, negative on a transport error. + * Errors are logged here so both public callers report them the same way. */ +static int _das_http_readChunk(DasHttpResp* pRes, char* pBuf, size_t uLen) +{ + int nRead = 0; + if(pRes->pSsl != NULL){ + nRead = SSL_read((SSL*)pRes->pSsl, pBuf, (int)uLen); + if(nRead < 0){ + char* sErr = das_ssl_getErr(pRes->pSsl, nRead); + daslog_error_v("Error reading from SSL socket, %s", sErr); + free(sErr); + } + } + else{ + errno = 0; + nRead = recv(pRes->nSockFd, pBuf, uLen, 0); + if(nRead < 0) + daslog_error_v("Error reading from socket, %s", strerror(errno)); + } + return nRead; +} + +/* Release the transport. das_http_getBody() creates the SSL object, so it + * is freed here and not by DasIO_close(), which only knows the socket. */ +static void _das_http_closeBody(DasHttpResp* pRes) +{ + if(pRes->pSsl != NULL){ + pRes->nSockFd = SSL_get_fd((SSL*)pRes->pSsl); + if(SSL_shutdown((SSL*)pRes->pSsl) == 0) /* 0 means two-stage shutdown */ + SSL_shutdown((SSL*)pRes->pSsl); + SSL_free((SSL*)pRes->pSsl); + pRes->pSsl = NULL; + } + if(pRes->nSockFd < 0) return; + + daslog_debug_v("Shutting down socket %d", pRes->nSockFd); +#ifndef _WIN32 + shutdown(pRes->nSockFd, SHUT_RDWR); + close(pRes->nSockFd); +#else + shutdown(pRes->nSockFd, SD_BOTH); + closesocket(pRes->nSockFd); +#endif + pRes->nSockFd = -1; +} + +/* ************************************************************************* */ +/* A just-give-me-a-bag-of-bytes convenience function */ + DasAry* das_http_readUrl( const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, int64_t nLimit, float rConSec ){ - if( ! das_http_getBody(sUrl, sAgent, pMgr, pRes, rConSec)){ + return das_http_readUrl_ex(sUrl, sAgent, pMgr, pRes, nLimit, rConSec, NULL); +} + +DasAry* das_http_readUrl_ex( + const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, + int64_t nLimit, float rConSec, const char* sReqHdrs +){ + if( ! das_http_getBody_ex(sUrl, sAgent, pMgr, pRes, rConSec, sReqHdrs)){ return NULL; } + if(pRes->nCode == HTTP_NotModified) return NULL; /* no body by design, see http.h */ if(nLimit < 1) nLimit = -1; int64_t nTotal = 0; DasAry* pAry = new_DasAry("http_body", vtUByte, 1, NULL, RANK_1(0), UNIT_DIMENSIONLESS); - SSL* pSsl = (SSL*)pRes->pSsl; char buf[D2CHAR_CHUNK_SZ]; int nRead = 0; - if(DasHttpResp_useSsl(pRes)){ - while((nLimit == -1)||(nTotal < nLimit)){ - nRead = SSL_read(pSsl, buf, D2CHAR_CHUNK_SZ); - - if(nRead == 0) break; /* Socket is done */ - - if(nRead < 1){ /* Socket is broke */ - char* sErr = das_ssl_getErr(pSsl, nRead); - daslog_error_v("Error reading from SSL socket, %s", sErr); - free(sErr); - dec_DasAry(pAry); - return false; - } - - nTotal += nRead; - if(!DasAry_append(pAry, (const ubyte*) buf, nRead)) return false; /* Yay data! */ + while((nLimit == -1)||(nTotal < nLimit)){ + nRead = _das_http_readChunk(pRes, buf, D2CHAR_CHUNK_SZ); + if(nRead == 0) break; /* Socket is done */ + if(nRead < 0){ /* Socket is broke */ + dec_DasAry(pAry); + pAry = NULL; + break; + } + nTotal += nRead; + if(!DasAry_append(pAry, (const ubyte*) buf, nRead)){ + dec_DasAry(pAry); + pAry = NULL; + break; } - - pRes->nSockFd = SSL_get_fd((SSL*)pRes->pSsl); - if(SSL_shutdown((SSL*)pRes->pSsl) == 0) // If 0 needs 2-stage shutdown - SSL_shutdown((SSL*)pRes->pSsl); - - SSL_free((SSL*)pRes->pSsl); - pRes->pSsl = NULL; } - else{ - while((nLimit == -1)||(nTotal < nLimit)){ - errno = 0; - nRead = recv(pRes->nSockFd, buf, D2CHAR_CHUNK_SZ, 0); - if(nRead == 0) break; /* Socket is done */ + _das_http_closeBody(pRes); + if(pAry == NULL) return NULL; - if(nRead < 0){ /* Socket is broke */ - daslog_error_v("Error reading from socket, %s", strerror(errno)); - dec_DasAry(pAry); - return false; - } - - nTotal += nRead; - if(!DasAry_append(pAry, (const ubyte*) buf, nRead)) return false; /* Yay data! */ - } - } - if(nTotal > nLimit){ + if((nLimit > 0)&&(nTotal > nLimit)){ daslog_warn_v("Limit of %" PRId64 " bytes hit, almost certainly returning a " "partial download", nLimit); } else daslog_debug_v("%ld bytes read from %s", DasAry_size(pAry), sUrl); - daslog_debug_v("Shutting down socket %d", pRes->nSockFd); -#ifndef _WIN32 - shutdown(pRes->nSockFd, SHUT_RDWR); - close(pRes->nSockFd); -#else - shutdown(pRes->nSockFd, SD_BOTH); - closesocket(pRes->nSockFd); -#endif - pRes->nSockFd = -1; - return pAry; } + +/* ************************************************************************* */ +/* Streaming counterpart, for bodies too big or too slow to hold in memory */ + +DasErrCode das_http_copyBody( + DasHttpResp* pRes, FILE* pOut, int64_t* pCopied, das_progress_f onProgress, + void* pUser +){ + DasErrCode nRet = DAS_OKAY; + int64_t nTotal = 0; + char buf[D2CHAR_CHUNK_SZ]; + int nRead = 0; + + while(true){ + nRead = _das_http_readChunk(pRes, buf, D2CHAR_CHUNK_SZ); + if(nRead == 0) break; + if(nRead < 0){ nRet = DASERR_HTTP; break; } + + errno = 0; + if(fwrite(buf, 1, nRead, pOut) != (size_t)nRead){ + daslog_error_v("Error writing message body, %s", strerror(errno)); + nRet = DASERR_IO; + break; + } + nTotal += nRead; + + /* The callback is the only way a caller can bail out of a long + * download, so its code is honored even mid-body and passed up + * unchanged for the caller to recognize */ + if(onProgress != NULL){ + nRet = onProgress(nTotal, pUser); + if(nRet != DAS_OKAY) break; + } + } + + _das_http_closeBody(pRes); + if(pCopied != NULL) *pCopied = nTotal; + return nRet; +} diff --git a/das3/http.h b/das3/http.h index 25f4d39..172d7fe 100644 --- a/das3/http.h +++ b/das3/http.h @@ -1,4 +1,4 @@ -/* Copyright (C) 2017 Chris Piker +/* Copyright (C) 2017 - 2026 Chris Piker * * This file is part of das2C, the Core Das2 C Library. * @@ -19,6 +19,7 @@ #define _das_http_h_ #include +#include #include #include @@ -33,10 +34,13 @@ extern "C" { * @{ */ +/* Sizes of the parts of struct das_url, not counting the terminator. A URL + * with a part longer than this is refused by DasHttpResp_init(), it is never + * sent truncated. */ #define DASURL_SZ_SCHEME 31 #define DASURL_SZ_HOST 63 -#define DASURL_SZ_PATH 127 -#define DASURL_SZ_QUERY 1023 +#define DASURL_SZ_PATH 255 +#define DASURL_SZ_QUERY 3071 #define DASURL_SZ_DATASET 63 #define DASURL_SZ_PORT 7 @@ -121,12 +125,12 @@ typedef struct das_http_response_t{ DAS_API char* das_url_toStr(const struct das_url* pUrl, char* sBuf, size_t uLen); + /** Initialize all fields in an http response to default values. * - * All char fields are set to zero, the file descriptor (nSockFd) is set to - * -1 and the response code is set to zero. This function is called - * automatically by http_getBodySocket() so there is usually no need to use it - * directly. + * All char fields are set to zero, the file descriptor (nSockFd) and the + * response code are set to -1. This function is called automatically by + * das_http_getBody() so there is usually no need to use it directly. * * @param pRes The response to clear * @memberof DasHttpResp @@ -144,9 +148,13 @@ DAS_API void DasHttpResp_freeFields(DasHttpResp* pRes); * * The URL will be parsed into a das_url structure and stored internally. * - * @param pRes The response to initialized + * @param pRes The response to initialize * @param sUrl The fully qualified URL including the scheme, host, port (if any), * path and query parameters. + * @returns true if every part fit its field in struct das_url. False, with + * pRes->sError set, if the scheme is not http or https, the host + * or port is bad, or a part is longer than its DASURL_SZ_ limit. + * Nothing is truncated. * @memberof DasHttpResp */ DAS_API bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl); @@ -157,6 +165,24 @@ DAS_API bool DasHttpResp_init(DasHttpResp* pRes, const char* sUrl); */ DAS_API bool DasHttpResp_useSsl(DasHttpResp* pRes); +/** Get the value of one header from the final response + * + * Field names match without regard to case, as HTTP requires. The value + * is copied with surrounding white space removed. Only the headers of the + * final response are searched, those of any redirect on the way are gone. + * + * @param sField The header name, for example "ETag" or "Last-Modified" + * @param sBuf Receives the value, always NUL terminated when uLen > 0 + * @param uLen Size of sBuf including the terminator + * @returns true if the header is present and its value fit, false if it is + * absent, the response has no headers yet, or the value would + * have been truncated + * @memberof DasHttpResp + */ +DAS_API bool DasHttpResp_header( + const DasHttpResp* pRes, const char* sField, char* sBuf, size_t uLen +); + /** Get a socket positioned at the start of a remote resource * * This function makes a connection to a remote HTTP or HTTPS server, reads @@ -181,14 +207,17 @@ DAS_API bool DasHttpResp_useSsl(DasHttpResp* pRes); * socket descriptor or the SSL connection when it is finished reading the * message body. * - * Das2 Note: Since das2 servers can request different authentication for - * each dataset, the get string is inspected for the 'server=dataset' pair. If - * found the URL saved in the credentials manager will be - * http://SERVER/path?dataset=DATASET instead of just http://SERVER/path. - * This a bit of a hack and a better solution should be found in the future. + * das2.2 Note: The das2.2 protocol lets a server require a different + * login for each dataset it serves (das2.2 ICD, https://zenodo.org/records/3588535). + * So when the query string carries a dataset= value, credentials are looked + * up and saved under http://SERVER/path?dataset=DATASET rather than + * http://SERVER/path. This is part of the das2.2 interface, which das2C + * supports alongside das3. * - * @param sUrl The location to connect to, will be URL encoded before - * transmission + * @param sUrl The location to connect to, sent as given. No encoding is + * applied here, so reserved characters in parameter values must + * already be percent encoded by the caller. Each part must fit + * its field in struct das_url, see DasHttpResp_init(). * * @param sAgent The user agent string you wish to send to the server. If NULL * then the string "libdas/3" is sent. @@ -214,6 +243,34 @@ DAS_API bool das_http_getBody( const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, float rConSec ); + +/** das_http_getBody() with extra request headers, for conditional requests + * + * The one use today is cache revalidation. A caller holding a copy of a + * resource along with the ETag or Last-Modified header it came with sends + * + * "If-None-Match: \"5a6-607e10c29b347\"\r\nIf-Modified-Since: Tue, 17 Oct 2023 03:23:33 GMT\r\n" + * + * and a server that still has that version answers 304 Not Modified with + * no body. Send both validators when both are known: a server with an + * ETag ignores If-Modified-Since (RFC 9110) and one without an ETag can + * only use the date. Values go back exactly as the server sent them, + * see DasHttpResp_header(). + * + * @param sReqHdrs Zero or more complete header lines, each ending in + * "\r\n", appended to the request as given. NULL or empty sends a + * plain request, which is what das_http_getBody() does. + * + * @returns As das_http_getBody(), plus one case: true with pRes->nCode + * 304 and no body socket, nSockFd -1 and pSsl NULL, when the + * server reports the copy is current. pRes->sHeaders holds the + * 304 response's headers. Callers that sent no validator never + * see this. + */ +DAS_API bool das_http_getBody_ex( + const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, + float rConSec, const char* sReqHdrs +); /** Read all the bytes for a URL into a byte array * @@ -261,6 +318,63 @@ DAS_API DasAry* das_http_readUrl( int64_t nLimit, float rConSec ); +/** das_http_readUrl() with extra request headers, see das_http_getBody_ex() + * + * @returns The body as das_http_readUrl() does, or NULL in two cases the + * caller tells apart by pRes->nCode: 304, the server says the + * caller's copy is current and sent no body, pRes->sError is NULL; + * anything else, the request failed and pRes->sError says why. + */ +DAS_API DasAry* das_http_readUrl_ex( + const char* sUrl, const char* sAgent, DasCredMngr* pMgr, DasHttpResp* pRes, + int64_t nLimit, float rConSec, const char* sReqHdrs +); + +/** Progress callback for das_http_copyBody() + * + * Follows the StreamHandler convention: a non-zero return stops the copy + * and becomes the return value of das_http_copyBody(). Use a code of your + * own rather than DASERR_HTTP or DASERR_IO, otherwise a deliberate stop + * can't be told from a transport or write failure. + * + * @param nBytes The number of body bytes written so far + * @param pUser The opaque pointer handed to das_http_copyBody() + * @returns DAS_OKAY to keep copying, anything else to stop + */ +typedef DasErrCode (*das_progress_f)(int64_t nBytes, void* pUser); + +/** Copy the message body of an open HTTP response to a FILE + * + * The streaming counterpart to das_http_readUrl(). Call das_http_getBody() + * first, then hand the response here to drain the socket, or SSL connection, + * into pOut. Only one chunk is held in memory at a time so bodies of any + * size may be copied. + * + * The connection is closed before returning, whether or not the copy + * succeeded, so afterwards pRes->nSockFd is -1 and pRes->pSsl is NULL. + * The remaining response fields are untouched, release them with + * DasHttpResp_freeFields(). + * + * Errors are logged using the functions in log.h in the manner of + * das_http_readUrl(), das_error() is not called. + * + * @param pRes A response for which das_http_getBody() returned true + * @param pOut Destination for the body bytes. No flush is performed. + * @param pCopied If not NULL, receives the number of bytes written to pOut. + * Set even on failure or an early stop. + * @param onProgress If not NULL, called after each chunk is written. A + * non-zero return stops the copy. + * @param pUser Passed through to onProgress untouched + * + * @returns DAS_OKAY if the whole body was copied, DASERR_HTTP if reading + * from the server failed, DASERR_IO if writing to pOut failed, or + * whatever onProgress returned if it stopped the copy. + */ +DAS_API DasErrCode das_http_copyBody( + DasHttpResp* pRes, FILE* pOut, int64_t* pCopied, das_progress_f onProgress, + void* pUser +); + /** @} */ #ifdef __cplusplus diff --git a/das3/io.h b/das3/io.h index d676493..2426d1d 100644 --- a/das3/io.h +++ b/das3/io.h @@ -236,7 +236,7 @@ DAS_API DasIO* new_DasIO_str(const char* sProg, char* sbuf, size_t len, const ch * * You should also setup the SSL connection with the SSL_MODE_AUTO_RETRY option * to SSL_set_mode or SSL_CTX_set_mode to let the openssl library handle - * session renegotiation transparently. The http_getBodySocket() call does + * session renegotiation transparently. The das_http_getBody() call does * initialize any SSL structures it generates in auto-retry mode. * * @param sProg A spot to store the name of the program creating the file diff --git a/das3/json.c b/das3/json.c index 3db06be..10ec45f 100644 --- a/das3/json.c +++ b/das3/json.c @@ -35,6 +35,7 @@ #include #include #include /* Only needed for DasJdo_get, could do manually */ +#include #include "util.h" #include "json.h" @@ -2191,9 +2192,7 @@ void* DasJdo_writeMinified(const DasJdo* pThis, size_t *out_size) { return 0; } - size += 1; // for the '\0' null terminating character - - data = (char *)malloc(size); + data = (char *)malloc(size + 1); // plus the '\0' null terminating character if (0 == data) { // malloc failed! @@ -2212,7 +2211,7 @@ void* DasJdo_writeMinified(const DasJdo* pThis, size_t *out_size) { *data_end = '\0'; if (0 != out_size) { - *out_size = size; + *out_size = size; // the length, not counting the terminator } return data; @@ -2527,9 +2526,7 @@ void *DasJdo_writePretty(const DasJdo* pThis, const char *indent, return 0; } - size += 1; // for the '\0' null terminating character - - data = (char *)malloc(size); + data = (char *)malloc(size + 1); // plus the '\0' null terminating character if (0 == data) { // malloc failed! @@ -2548,7 +2545,7 @@ void *DasJdo_writePretty(const DasJdo* pThis, const char *indent, *data_end = '\0'; if (0 != out_size) { - *out_size = size; + *out_size = size; // the length, not counting the terminator } return data; @@ -2656,13 +2653,19 @@ const DasJdo* DasJdo_get(const DasJdo* pThis, const char* sRelPath){ size_t uLen = (*pSep == '\0') ? strlen(sRelPath) : pSep - sRelPath; - if((uLen == 0)||(uLen == 1)) return NULL; /* Invalid relative path */ + if(uLen == 0) return NULL; /* Invalid relative path */ bool bRecurse = ((*pSep != '\0') && (*(pSep+1) != '\0')); - /* Convert string to number since integers are used to lookup array values - * and can also be used to find dictionary values */ + /* A segment of digits is an index, usable for arrays and dictionaries + * alike. Only this segment counts, "0/EMAIL" is index 0 then EMAIL. */ int iItem = -1; - if( ! das_str2int(sRelPath, &iItem) ) iItem = -1; + if(uLen < 10){ + iItem = 0; + for(size_t u = 0; u < uLen; ++u){ + if(!isdigit((unsigned char)sRelPath[u])){ iItem = -1; break; } + iItem = iItem*10 + (sRelPath[u] - '0'); + } + } /* Really should avoid linear searches like this */ int i = 0; diff --git a/das3/json.h b/das3/json.h index dafad21..e87cdef 100644 --- a/das3/json.h +++ b/das3/json.h @@ -66,7 +66,14 @@ extern "C" { #endif -/** @addtogroup catalog +/** @defgroup json JSON + * A parser for JSON text and accessors for the resulting document tree. + * + * Catalog nodes happen to be JSON documents, so DasNode_getJdo() returns + * these objects, but nothing here is specific to catalogs. + */ + +/** @addtogroup json * @{ */ @@ -427,10 +434,12 @@ DAS_API const das_json_ary* DasJdo_array(const DasJdo* pThis); * @param pThis The top level document object to write, all sub-objects will * be written as well. * + * @param out_size If not NULL, receives the length of the string in bytes, + * not counting the null terminator, as strlen() would report. + * * @return A new buffer containing utf-8 string data, or NULL if an error * occurred (malformed JSON input, or malloc failed), * - * The out_size parameter is optional as the utf-8 string is null terminated. * @memberof DasJdo */ DAS_API void* DasJdo_writeMinified(const DasJdo* pThis, size_t *out_size); @@ -447,8 +456,11 @@ DAS_API void* DasJdo_writeMinified(const DasJdo* pThis, size_t *out_size); * * DasJdo_writePretty performs 1 call to malloc for the entire encoding. * + * @param out_size If not NULL, receives the length of the string in bytes, + * not counting the null terminator, as strlen() would report. + * * @return NULL if an error occurred (malformed JSON input, or malloc failed). - * The out_size parameter is optional as the utf-8 string is null terminated. + * @memberof DasJdo */ DAS_API void* DasJdo_writePretty(const DasJdo* pThis, const char *indent, const char *newline, size_t *out_size); diff --git a/das3/node.c b/das3/node.c index 502324a..b746fc9 100644 --- a/das3/node.c +++ b/das3/node.c @@ -15,34 +15,38 @@ * version 2.1 along with libdas2; if not, see . */ +/* Current upgrades provided here (2026-10-27) + + 1. JSON Collection objects (v0.5) are given mime type data + + no others at this time +*/ + #define _POSIX_C_SOURCE 200112L -#include -#include #include +#include #ifndef _WIN32 -#include +#include #else -#include +#include +#define getcwd _getcwd #endif -#include - #include "util.h" #include "log.h" -#include "http.h" #include "array.h" #include "node.h" #include "json.h" /* Very nice lib from https://github.com/sheredom/json.h * customized for local use */ -#include "das3/node.h" +#include "node_priv.h" /* Time out values for connections */ -/* Built-in catalog roots. */ +/* The compiled in seed: the index of the das catalog and its mirrors */ static const char* g_lDasDefRoots[2] = { "https://das2.org/catalog/index.json", "https://raw.githubusercontent.com/das-developers/das-cat/master/cat/index.json" @@ -50,37 +54,14 @@ static const char* g_lDasDefRoots[2] = { static int g_nDasDefRoots = 2; -const char** das_root_urls(size_t* pLen){ +const char** das_seed_urls(size_t* pLen){ if(pLen != NULL) *pLen = g_nDasDefRoots; return g_lDasDefRoots; } -#define D2C_LOCAL_SUB_SZ 32 - -/* Catalog nodes have extra arrays to cache nodes */ -#define _D2C_SEP_SZ 8 -typedef struct das_catalog { - DasNode base; - - DasAry* pSubNodes; - - DasAry* pSubPaths; - - /* The json object or xml element tag which contains the sub elements */ - void* pContainer; - - /* The path separator, defaults to '/' unless overridden by the object */ - char sPathSep[_D2C_SEP_SZ]; - - /* The name of the element which contains sub elements */ - char sContainer[64]; - -} DasCatNode; - - -static das_node_type_e das_node_type(const char* sType){ +static das_node_type_e _dasnode_type(const char* sType){ if(sType == NULL) return d2node_inv; if(strcmp(sType, D2CV_TYPE_CATALOG) == 0) return d2node_catalog; if(strcmp(sType, D2CV_TYPE_COLLECTION) == 0) return d2node_collection; @@ -88,7 +69,7 @@ static das_node_type_e das_node_type(const char* sType){ if(strcmp(sType, D2CV_TYPE_STREAM) == 0) return d2node_stream_src; if(strcmp(sType, D2CV_TYPE_TIMEAGG) == 0) return d2node_file_agg; if(strcmp(sType, D2CV_TYPE_SPASE) == 0) return d2node_spase_cat; - if(strcmp(sType, D2Cv_TYPE_SPDF_MASTER) == 0) return d2node_spdf_cat; + if(strcmp(sType, D2CV_TYPE_SPDF_MASTER) == 0) return d2node_spdf_cat; return d2node_inv; } @@ -104,6 +85,24 @@ bool DasNode_isCatalog(const DasNode* pThis) ); } +/* These four were declared in node.h since 2018 but never defined, so any + program that called them failed to link. das_fetch is the first. */ +bool DasNode_isStreamSrc(const DasNode* pThis){ + return (pThis != NULL) && (pThis->nType == d2node_stream_src); +} + +bool DasNode_isSpaseRec(const DasNode* pThis){ + return (pThis != NULL) && (pThis->nType == d2node_spase_cat); +} + +bool DasNode_isSpdfCat(const DasNode* pThis){ + return (pThis != NULL) && (pThis->nType == d2node_spdf_cat); +} + +das_node_type_e DasNode_type(const DasNode* pThis){ + return (pThis != NULL) ? pThis->nType : d2node_inv; +} + bool DasNode_isJson(const DasNode* pThis){ return ( (pThis->nType == d2node_catalog)||(pThis->nType == d2node_stream_src) @@ -126,7 +125,8 @@ const DasJdo* DasNode_getJdo(const DasNode* pThis, const char* sFragment) } -const DasJdo* DasNode_getJdoType( +/* Was public until 2026, nothing outside this file ever called it */ +static const DasJdo* _DasNode_getJdoType( const DasNode* pThis, enum das_json_type_e type, const char* sFragment ){ const DasJdo* pObj = DasNode_getJdo(pThis, sFragment); @@ -142,77 +142,228 @@ const char* DasNode_pathUri(const DasNode* pThis) } /* helper, git a string value from the root of the dom */ -static const char* DasNode_rootStr(const DasNode* pThis, const char* sStr) +static const char* _DasNode_rootStr(const DasNode* pThis, const char* sStr) { if(!DasNode_isJson(pThis)){ daslog_error("Non-JSON nodes not supported at this time"); return NULL; } - const DasJdo* pObj = DasNode_getJdoType(pThis, das_json_type_str, sStr); - if(pObj == NULL){ - daslog_error_v("Error in node from %s, '%s' missing or not a string", - pThis->sURL, sStr); + /* Most root strings (title, description, version) are optional, so a + miss is the caller's business, not an error. Callers that require a + field check for NULL and complain themselves. */ + const DasJdo* pObj = _DasNode_getJdoType(pThis, das_json_type_str, sStr); + if(pObj == NULL) return NULL; - } return ((das_json_str*)pObj->value)->string; } const char* DasNode_name(const DasNode* pThis){ - return DasNode_rootStr(pThis, D2FRAG_NAME); + return _DasNode_rootStr(pThis, D2FRAG_NAME); } -static const char* DasNode_label(const DasNode* pThis){ - const char* pLabel = DasNode_rootStr(pThis, D2FRAG_LABEL); +const char* DasNode_label(const DasNode* pThis){ + const char* pLabel = _DasNode_rootStr(pThis, D2FRAG_LABEL); if(pLabel != NULL) return pLabel; - return DasNode_rootStr(pThis, D2FRAG_NAME); + return _DasNode_rootStr(pThis, D2FRAG_NAME); } const char* DasNode_title(const DasNode* pThis){ - return DasNode_rootStr(pThis, D2FRAG_TITLE); + return _DasNode_rootStr(pThis, D2FRAG_TITLE); } const char* DasNode_summary(const DasNode* pThis){ - return DasNode_rootStr(pThis, D2FRAG_DESC); + return _DasNode_rootStr(pThis, D2FRAG_DESC); } const char* DasNode_srcUrl(const DasNode* pThis){ return pThis->sURL;} +const char* DasNode_version(const DasNode* pThis){ + return _DasNode_rootStr(pThis, D2FRAG_VERSION); +} + +const char* DasNode_pathSep(const DasNode* pThis){ + if(!DasNode_isCatalog(pThis)) return NULL; + return ((const DasCatNode*)pThis)->sPathSep; +} + /* ************************************************************************* */ -/* Direct Node Construction */ +/* Child listings. These read the container dictionary that was downloaded + * with the node, so they never touch the network. Linear walks of a linked + * list, fine for the few dozen entries real catalogs have. */ -/* make a node from a *specific* URL, does not call das_error functions unless - * a memory allocation error occurs because network operations are assumed to - * fail sometimes. Returns a new node allocated on the heap if the URL - * contained a recognizable DasCatalog object - */ -static DasNode* _DasNode_mkNode( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, - const char* sAgent, float rConSec -){ - /* Read JSON bytes into the array, put a sanity limit of 20 MB on the read, - * no catalogs should ever get anywhere near that size, but you don't know */ - DasHttpResp httpRes; - DasAry* pBytesAry = das_http_readUrl( - sUrl, sAgent, pMgr, &httpRes, 1024*1024*20, rConSec - ); - if(pBytesAry == NULL){ - daslog_warn_v("%s", httpRes.sError); +size_t DasNode_subCount(const DasNode* pThis) +{ + if(!DasNode_isCatalog(pThis)) return 0; + const DasJdo* pDir = (const DasJdo*) ((const DasCatNode*)pThis)->pContainer; + if((pDir == NULL)||(pDir->type != das_json_type_dict)) return 0; + return ((const das_json_dict*)pDir->value)->length; +} + +static const das_json_dict_el* _DasNode_subEl(const DasNode* pThis, size_t uIdx) +{ + if(!DasNode_isCatalog(pThis)) return NULL; + const DasJdo* pDir = (const DasJdo*) ((const DasCatNode*)pThis)->pContainer; + if((pDir == NULL)||(pDir->type != das_json_type_dict)) return NULL; + + const das_json_dict_el* pEl = DasJdo_dictFirst(pDir); + for(size_t u = 0; (u < uIdx) && (pEl != NULL); ++u) pEl = pEl->next; + return pEl; +} + +const char* DasNode_subName(const DasNode* pThis, size_t uIdx) +{ + const das_json_dict_el* pEl = _DasNode_subEl(pThis, uIdx); + return pEl ? pEl->name->string : NULL; +} + +const DasJdo* DasNode_subStub(const DasNode* pThis, size_t uIdx) +{ + const das_json_dict_el* pEl = _DasNode_subEl(pThis, uIdx); + if((pEl == NULL)||(pEl->value == NULL)||(pEl->value->type != das_json_type_dict)) return NULL; - } + return pEl->value; +} + +das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx) +{ + const DasJdo* pStub = DasNode_subStub(pThis, uIdx); + if(pStub == NULL) return d2node_inv; + return _dasnode_type( DasJdo_string( DasJdo_get(pStub, D2FRAG_TYPE) ) ); +} + +/* ************************************************************************* */ +/* Presenting every document in the current layout + * + * Past this point the library reads one document model, the current + * catalog layout. Each entry in the table below recognizes one older + * shape by node type and version and rewrites the parsed DOM to match. + * Elements the parser did not allocate come from a side block, one per + * node, freed beside the DOM. + * + * If a document format other than JSON is ever read (Das2DSDF text, SPASE + * XML), this table is where it goes: emit JSON text in the current layout + * and run it through the parser, so nothing downstream learns a second + * model. None is written; this only says where. */ + +/* ---- 2018 Collection shim, begin. When those catalogs are regenerated + delete down to the end marker, plus the _dasnode_upgrade() call in + _dasnode_fromJson() and the pShim field: a table with no entries is + not C99, so the next translator brings the table back. The comment + above stays; it says where. ---- */ + +static bool _dasnode_ieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + for(; *a && *b; ++a, ++b) + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + return *a == *b; +} - /* Save the final URL, may be needed for error messages, may not match - * sUrl due to re-directs */ - char sFinalUrl[256] = {'\0'}; - das_url_toStr(&(httpRes.url), sFinalUrl, 511); +/* 2018 Collections, version 0.5 and earlier, describe each source by a + convention string instead of the media types it provides. The list + the convention implies is appended to the entry so readers need only + one rule. Delete this entry when those catalogs are regenerated. */ + +struct provides_patch { /* "provides": ["mime"], all in one block */ + das_json_dict_el el; + das_json_str name; + DasJdo aryVal; + das_json_ary ary; + das_json_ary_el aryEl; + DasJdo strVal; + das_json_str mime; +}; + +static const char* _dasnode_impliedProvides(const DasJdo* pEntry) +{ + if((pEntry == NULL)||(pEntry->type != das_json_type_dict)) return NULL; + if(!_dasnode_ieq(DasJdo_string(DasJdo_get(pEntry, D2FRAG_TYPE)), D2CV_TYPE_STREAM)) return NULL; + if(DasJdo_get(pEntry, "provides") != NULL) return NULL; + const char* sConv = DasJdo_string(DasJdo_get(pEntry, "convention")); + if(_dasnode_ieq(sConv, "das2/2.2")) return "application/vnd.das2.das2stream"; + if(_dasnode_ieq(sConv, "hapi/2.0")) return "application/vnd.hapi.stream"; + return NULL; +} + +/* TODO: Remove this particular upgrade function once no old style collection + catalog nodes exist in the wild */ +static bool _dasnode_upgradeCollection(DasJdo* pRoot, void** ppShim) +{ + const DasJdo* pSrcs = DasJdo_get(pRoot, D2FRAG_SOURCES); + if((pSrcs == NULL)||(pSrcs->type != das_json_type_dict)) return true; /* reported later */ + + size_t uNeed = 0; + const das_json_dict_el* pEl = NULL; + for(pEl = DasJdo_dictFirst(pSrcs); pEl; pEl = pEl->next) + if(_dasnode_impliedProvides(pEl->value) != NULL) ++uNeed; + if(uNeed == 0) return true; + struct provides_patch* aPatch = (struct provides_patch*)calloc( + uNeed, sizeof(struct provides_patch) + ); + if(aPatch == NULL) return false; + *ppShim = aPatch; + + size_t u = 0; + for(pEl = DasJdo_dictFirst(pSrcs); pEl; pEl = pEl->next){ + const char* sMime = _dasnode_impliedProvides(pEl->value); + if(sMime == NULL) continue; + struct provides_patch* p = aPatch + u++; + p->name.string = "provides"; p->name.string_size = 8; + p->mime.string = sMime; p->mime.string_size = strlen(sMime); + p->strVal.type = das_json_type_str; p->strVal.value = &p->mime; + p->aryEl.value = &p->strVal; + p->ary.start = &p->aryEl; p->ary.length = 1; + p->aryVal.type = das_json_type_ary; p->aryVal.value = &p->ary; + p->el.name = &p->name; p->el.value = &p->aryVal; + + das_json_dict* pDict = (das_json_dict*)pEl->value->value; + if(pDict->start == NULL){ + pDict->start = &p->el; + } + else{ + das_json_dict_el* pLast = pDict->start; + while(pLast->next != NULL) pLast = pLast->next; + pLast->next = &p->el; + } + pDict->length += 1; + } + return true; +} + +static const struct { + das_node_type_e nType; + double rThrough; /* versions up to this one */ + bool (*fn)(DasJdo* pRoot, void** ppShim); +} g_upgrades[] = { + { d2node_collection, 0.5, _dasnode_upgradeCollection }, +}; - /* Parse the JSON data */ - size_t uBytes; - const char* pBytes = (const char*) (DasAry_getBytesIn(pBytesAry, DIM0, &uBytes)); +/* False only when memory ran out. An unversioned document counts as the + oldest, since versions were added after these layouts. */ +static bool _dasnode_upgrade(DasJdo* pRoot, das_node_type_e nType, void** ppShim) +{ + const char* sVer = DasJdo_string(DasJdo_get(pRoot, D2FRAG_VERSION)); + double rVer = (sVer == NULL) ? 0.0 : das_strtod_c(sVer, NULL); + for(size_t u = 0; u < sizeof(g_upgrades)/sizeof(g_upgrades[0]); ++u){ + if((g_upgrades[u].nType != nType)||(rVer > g_upgrades[u].rThrough)) continue; + if(!g_upgrades[u].fn(pRoot, ppShim)) return false; + } + return true; +} +/* ---- 2018 Collection shim, end ---- */ + +/* Build a node from JSON text. sUrl is where it came from and sFinalUrl + * the same after redirects, both for messages. sPathUri may be NULL. The + * node is not marked as owned; constructors decide that. */ +DasNode* _dasnode_fromJson( + const char* pBytes, size_t uBytes, const char* sUrl, const char* sFinalUrl, + const char* sPathUri +){ if(uBytes < 2){ daslog_error_v("String data from %s too small to be a valid JSON document", sUrl); return NULL; @@ -224,9 +375,6 @@ static DasNode* _DasNode_mkNode( pRoot = das_json_parse_ex(pBytes, uBytes, das_jparse_flags_allow_json5, NULL, NULL, &jsonRes); - dec_DasAry(pBytesAry); /* Done with the byte array, data in json buffer now */ - DasHttpResp_freeFields(&httpRes); /* Done with the headers etc. */ - if(pRoot == NULL){ char sTmp[256] = {'\0'}; daslog_error_v("Error parsing JSON data for URL %s: %s", sFinalUrl, @@ -242,7 +390,7 @@ static DasNode* _DasNode_mkNode( return NULL; } - das_node_type_e nType = das_node_type( sType ); + das_node_type_e nType = _dasnode_type( sType ); if( nType == d2node_inv){ daslog_error_v("Error in catalog object from %s, 'TYPE' value %s is " "unknown", sFinalUrl, sType); @@ -250,13 +398,21 @@ static DasNode* _DasNode_mkNode( return NULL; } + /* Older layouts are rewritten here so the rest of the node code, and + everything built on it, reads only the current one. */ + void* pShim = NULL; + if(!_dasnode_upgrade(pRoot, nType, &pShim)){ + free(pRoot); + return NULL; + } + /* Initialize non-container types */ DasNode* pBase = NULL; if((nType == d2node_stream_src)||(nType == d2node_file_agg)){ pBase = (DasNode*)calloc(1, sizeof(DasNode)); if(pBase == NULL){ das_error(DASERR_NODE, "Couldn't allocate node"); - free(pRoot); + free(pRoot); free(pShim); return NULL; } } @@ -272,14 +428,14 @@ static DasNode* _DasNode_mkNode( (pVal->type != das_json_type_dict) ){ daslog_error_v("Error in catalog object from %s, missing dictionary " "element '%s'", sFinalUrl, sContainer); - free(pRoot); + free(pRoot); free(pShim); return NULL; } DasCatNode* pThis = (DasCatNode*)calloc(1, sizeof(DasCatNode)); if(pThis == NULL){ das_error(DASERR_NODE, "Couldn't allocate catalog node"); - free(pRoot); + free(pRoot); free(pShim); return NULL; } pBase = (DasNode*)pThis; @@ -312,21 +468,12 @@ static DasNode* _DasNode_mkNode( pBase->nType = nType; strncpy(pBase->sURL, sUrl, 511); pBase->pDom = pRoot; + pBase->pShim = pShim; if(sPathUri != NULL) strncpy(pBase->sPath, sPathUri, 511); return pBase; } -/* ************************************************************************* */ -/* Sub Node Construction */ - -DasNode* DasNode_subNode( - DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -); - -static void del_NonRootNode(DasNode* pThis); - - /* Helper for loadSubNode and subNode * * We have a fundamental difficulty in looking for sub nodes. Since a catalog @@ -372,7 +519,7 @@ static void del_NonRootNode(DasNode* pThis); * @param sSubPath - The sub path *WITH THE SEPARATOR ATTACHED* * */ -static bool _checkSubMatch(DasNode* pBase, const char* sSubPathWithSep) +bool _DasNode_subMatch(DasNode* pBase, const char* sSubPathWithSep) { if(!DasNode_isCatalog(pBase)) return false; @@ -381,12 +528,12 @@ static bool _checkSubMatch(DasNode* pBase, const char* sSubPathWithSep) const char* sSep = "/"; - const DasJdo* pObj = DasNode_getJdoType(pBase, das_json_type_str, D2FRAG_PATH_SEP); + const DasJdo* pObj = _DasNode_getJdoType(pBase, das_json_type_str, D2FRAG_PATH_SEP); if(pObj != NULL){ sSep = ((das_json_str*)pObj->value)->string; } else{ - pObj = DasNode_getJdoType(pBase, das_json_type_null, D2FRAG_PATH_SEP); + pObj = _DasNode_getJdoType(pBase, das_json_type_null, D2FRAG_PATH_SEP); if(pObj != NULL) sSep = ""; } @@ -422,339 +569,638 @@ static bool _checkSubMatch(DasNode* pBase, const char* sSubPathWithSep) /* sRelPath is the desired sub path from the point of this object. If it's * NULL, that means you just wanted what you already have (pThis) so there's * no point in calling this function */ -static DasNode* _DasNode_loadSubNode_dasCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, - const char* sAgent + +DasNode* new_DasNode_json( + const char* pBytes, size_t uBytes, const char* sUrl, const char* sPathUri ){ - if(!DasNode_isJson((DasNode*)pThis)){ - daslog_error("XML catalogs not yet supported"); - return NULL; - } - const DasJdo* pDir = (DasJdo*) pThis->pContainer; + /* No name given means a root: the node has no URI and what is walked + to below it is named by the path from it */ + char sUri[600] = {'\0'}; + if((sPathUri != NULL) && (sPathUri[0] != '\0')) _das_uri_expand(sPathUri, sUri, sizeof(sUri)); + if(sUrl == NULL) sUrl = ""; + + DasNode* pNode = _dasnode_fromJson(pBytes, uBytes, sUrl, sUrl, sUri); + if(pNode) pNode->bOwner = true; + return pNode; +} - if((pDir == NULL) || (pDir->type != das_json_type_dict)){ - daslog_error_v("Catalog container node missing for node from %s", - pThis->base.sURL); - return NULL; +/* ************************************************************************* */ +/* Selective Destruction */ + +/* Frees a node and everything below it, except nodes a constructor handed + to a caller: those are freed only by del_DasNode(). Not public: a child + lives in its parent's pSubNodes cache, freeing one alone would leave a + dangling entry there. */ +void _DasNode_delTree(DasNode* pThis) +{ + if(pThis == NULL) return; + if(pThis->bOwner) return; + + /* Cascade through the children */ + if(DasNode_isCatalog(pThis)){ + DasCatNode* pCat = (DasCatNode*)pThis; + + ptrdiff_t uNodes = DasAry_lengthIn(pCat->pSubNodes, DIM0); + for(ptrdiff_t u = 0; u < uNodes; ++u){ + DasNode** ppSub = (DasNode**)DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(u)); + _DasNode_delTree(*ppSub); + } + dec_DasAry(pCat->pSubNodes); + dec_DasAry(pCat->pSubPaths); } + /* Every node holds a JSON DOM, see _dasnode_upgrade(); the shim holds + whatever was added to it on load. */ + free(pThis->pDom); + free(pThis->pShim); - /* Run over all the keys and see if any of them match the front of the - * requested relative path. If so try all the URLs given for full object - * realization before giving up */ - const das_json_dict_el* pEl = NULL; - size_t uLen; - const DasJdo* pChild = NULL; - const char* sChild = NULL; - const DasJdo* pChildUrls = NULL; - const das_json_ary_el* pUrlEl; - char sSubUri[560] = {'\0'}; - const char* sUrl = NULL; - const char* sSubRelPath = NULL; - DasNode* pNode = NULL; - DasNode* pDecendent = NULL; - int nUrl; - - /* Trying it implement a tree search that will auto backout and try again - * if a URL turns out to be broken, or contains bad data, or is the wrong - * child. */ - for(pEl = DasJdo_dictFirst(pDir); pEl != NULL; pEl = pEl->next){ - sChild = pEl->name->string; + free(pThis); +} +void del_DasNode(DasNode* pThis) +{ + if(pThis == NULL) return; + if(!pThis->bOwner) return; /* a child, its parent frees it */ - uLen = strlen(sChild); + pThis->bOwner = false; /* so the tree pass takes this node too */ + _DasNode_delTree(pThis); +} - /* See if this child and the requested item have the same chars up to - * the length of the child */ - if(strncmp(sChild, sRelPath, uLen) != 0) continue; +/* ************************************************************************* */ +/* URI spellings, see node_priv.h */ - pChild = pEl->value; /* If this never get's set, then we had no - * children with matching names */ +static const char* g_aUriShort[] = {"site:", "test:"}; - /* Just stop using this catalog node if it has broken syntax */ - if((pChild == NULL )||(pChild->type != das_json_type_dict)){ - daslog_error_v("Catalog error @ %s: sub item '%s' is not a dictionary", - pThis->base.sURL, pEl->name->string); - return NULL; - } +const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut) +{ + if(uOut == 0) return sOut; + sOut[0] = '\0'; + if(sIn == NULL) return sOut; + for(size_t u = 0; u < sizeof(g_aUriShort)/sizeof(g_aUriShort[0]); ++u){ + size_t uLen = strlen(g_aUriShort[u]); + if(strncmp(sIn, g_aUriShort[u], uLen) != 0) continue; + /* The shorthand alone names the node, "tag:das2.org,2012:site"; + with more after it, it stands for the node plus its ":/" child + separator, as the live catalog composes names top down */ + const char* sRest = sIn + uLen; + while(*sRest == '/') ++sRest; + if(*sRest == '\0') + snprintf(sOut, uOut, D2URI_ROOT "%.*s", (int)(uLen - 1), g_aUriShort[u]); + else + snprintf(sOut, uOut, D2URI_ROOT "%s/%s", g_aUriShort[u], sRest); + return sOut; + } + strncpy(sOut, sIn, uOut - 1); + sOut[uOut - 1] = '\0'; + return sOut; +} - pChildUrls = DasJdo_get(pChild, D2FRAG_URLS); - if((pChildUrls == NULL)||(pChildUrls->type != das_json_type_ary)){ - daslog_error_v("From %s: %s element of node '%s' doesn't " - "have a URLS array", pThis->base.sURL, sChild, - pThis->sContainer); - return NULL; - } +/* ************************************************************************* */ +/* The mount table */ - /* Loop over URLs for child */ - nUrl = -1; - pNode = NULL; - - /* this is a slow time out.. */ - float rConSec = DASHTTP_TO_MIN * DASHTTP_TO_MULTI; - - for(pUrlEl = DasJdo_aryFirst(pChildUrls); pUrlEl != NULL; - pUrlEl = pUrlEl->next){ - pNode = NULL; - nUrl += 1; - sUrl = DasJdo_string(pUrlEl->value); - if(sUrl == NULL){ - daslog_error_v("From %s: %s/%s/%d element is not a string", - pThis->base.sURL, D2FRAG_URLS, sChild, nUrl); - continue; - } +/* Append a JSON string literal, escaping the two characters that matter */ +static size_t _dasnode_jsonStr(char* sOut, size_t uOut, const char* sIn) +{ + size_t u = 0; + if(u < uOut - 1) sOut[u++] = '"'; + for(; (*sIn != '\0') && (u < uOut - 3); ++sIn){ + if((*sIn == '"')||(*sIn == '\\')) sOut[u++] = '\\'; + sOut[u++] = *sIn; + } + if(u < uOut - 1) sOut[u++] = '"'; + sOut[u] = '\0'; + return u; +} - /* tell sub-item it's name */ - snprintf(sSubUri, 559, "%s%s%s", pThis->base.sPath, pThis->sPathSep, sChild); - - /* TODO: Split mkNode and server contact into two separate functions - * so that we know if we can try again with a slow server, of if - * we just received bad data and shouldn't waste time reading it - * again */ - pNode = _DasNode_mkNode(sUrl, sSubUri, pMgr, sAgent, rConSec); - if(pNode == NULL) continue; - - /* If after removing the child portion from the relative path there - * is nothing left we're done. */ - sSubRelPath = sRelPath + uLen; - - if(sSubRelPath[0] == '\0'){ - DasAry_append(pThis->pSubNodes, (const ubyte*) &pNode, 1); - DasAry_append(pThis->pSubPaths, (const ubyte*) sChild, strlen(sChild)+1); - DasAry_markEnd(pThis->pSubPaths, DIM1); - return pNode; - } - - /* There's more, see if it's possible for this child to complete the - * path. */ - if(_checkSubMatch(pNode, sSubRelPath)){ - - pDecendent = DasNode_subNode((DasNode*)pNode, sSubRelPath, pMgr, sAgent); - if(pDecendent){ - /* Worked okay, cache the child node, but return the descendant - * node (however far down it came from */ - DasAry_append(pThis->pSubNodes, (const ubyte*) &pNode, 1); - DasAry_append(pThis->pSubPaths, (const ubyte*) sChild, strlen(sChild)+1); - DasAry_markEnd(pThis->pSubPaths, DIM1); - - return pDecendent; - } - } - else{ - /* This child can't match, stop checking this URL set*/ - del_NonRootNode(pNode); - pNode = NULL; - } +/* read comments in node_priv.h header on this function first before editing it */ +DasNode* _dasnode_newMountTable(const char** aUrl, const char** aUri, size_t uMounts) +{ + if(uMounts == 0){ + daslog_error("A mount table needs at least one mount"); + return NULL; + } + size_t uTotal = 0; + for(size_t i = 0; i < uMounts; ++i){ + if((aUri[i] == NULL)||(aUri[i][0] == '\0')||(aUrl[i] == NULL)||(aUrl[i][0] == '\0')){ + daslog_error("A mount needs both a URL to load and a URI to sit at"); + return NULL; } - - /* Must have been a look-alike child (or some other failure), try again */ - del_NonRootNode(pNode); - pNode = NULL; + uTotal += strlen(aUri[i]) + strlen(aUrl[i]); } - - daslog_error_v( - "Node %s (URI '%s') has no child node that starts with %s", - DasNode_label((DasNode*)pThis), DasNode_pathUri((DasNode*)pThis), sRelPath + /* The mount table is an ordinary Catalog node that only exists in memory. + * Its separator is null, like the das catalog index's, so child names + * are whole URI prefixes, and a lookup takes the first child whose name + * is a prefix of the path. + * + * Sized from the inputs since a mount list has no natural bound. The + * escaping below at most doubles a string, hence the factor. */ + size_t uJson = 256 + 2 * uTotal + 64 * uMounts; + char* sJson = (char*)calloc(uJson, 1); + if(sJson == NULL){ + das_error(DASERR_NODE, "Couldn't allocate the mount table"); + return NULL; + } + size_t u = 0; + u += snprintf(sJson + u, uJson - u, + "{\"type\":\"Catalog\",\"version\":\"0.5\",\"name\":\"mounts\"," + "\"title\":\"Mount table\",\"separator\":null,\"catalog\":{" ); - return NULL; + /* Longest URI first, see node_priv.h. A selection sort over an index + * array; mount lists are a handful of entries */ + size_t aOrder[64]; + if(uMounts > sizeof(aOrder)/sizeof(aOrder[0])){ + daslog_error("Too many mounts"); + free(sJson); + return NULL; + } + for(size_t i = 0; i < uMounts; ++i) aOrder[i] = i; + for(size_t i = 0; i + 1 < uMounts; ++i){ + size_t iMax = i; + for(size_t j = i + 1; j < uMounts; ++j) + if(strlen(aUri[aOrder[j]]) > strlen(aUri[aOrder[iMax]])) iMax = j; + size_t t = aOrder[i]; aOrder[i] = aOrder[iMax]; aOrder[iMax] = t; + } + + for(size_t i = 0; i < uMounts; ++i){ + size_t k = aOrder[i]; + if(i > 0) u += snprintf(sJson + u, uJson - u, ","); + u += _dasnode_jsonStr(sJson + u, uJson - u, aUri[k]); + u += snprintf(sJson + u, uJson - u, ":{\"type\":\"Catalog\",\"urls\":["); + u += _dasnode_jsonStr(sJson + u, uJson - u, aUrl[k]); + u += snprintf(sJson + u, uJson - u, "]}"); + } + u += snprintf(sJson + u, uJson - u, "}}"); + if(u >= uJson - 1){ + daslog_error("Mount URI or URL too long"); + free(sJson); + return NULL; + } + + /* Nothing is fetched here, the first walk through the table does that */ + DasNode* pTable = _dasnode_fromJson(sJson, u, "memory:mounts", "memory:mounts", ""); + free(sJson); + if(pTable != NULL) pTable->bOwner = true; + return pTable; } -static DasNode* _DasNode_loadSubNode_spdfCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -){ - daslog_error("SPDF catalogs are not yet supported"); - return NULL; +/* ************************************************************************* */ +/* Walking: resolving a path below a node, one document at a time + * + * The walk is the old recursive loader turned into an explicit stack, one + * frame per path level, with the fetch taken out. A frame searches one + * container for the child that starts the rest of the path, first among + * the children already loaded under it, then in its listing. A listing + * entry names one or more URLs; they are the candidates of a fetch step, + * offered in catalog order. A document that arrives is parsed and either + * finishes the walk, becomes the parent of the next frame, or is dropped + * because nothing under it could continue the path, in which case the + * next candidate is offered. When a frame runs out of candidates and + * children the walk backs out to the frame above, which drops the node it + * descended into and moves to its next candidate. + * + * First match wins at every level, as it always has: for the path "ab/x" + * a child named "a" is tried before "ab", and the separator check in + * _DasNode_subMatch() is what sends the walk on to "ab". A listing entry + * whose name is already among the loaded children is skipped, since the + * loaded copy was tried first and a second copy would only duplicate it. + * + * Nothing attaches to the tree until the walk succeeds. Then every node + * it loaded joins its parent's child list and the walk forgets them. + * + * The seed walk, new_DasNodeWalk_seed(), has one frame with no parent + * whose candidates are the compiled in seed URLs. The node it returns is + * a tree top the caller owns, and lookups by full URI are walks from it; + * a session keeps it for its lifetime, see catalog.c. */ + +#define _WALK_DEPTH 64 /* more levels than any catalog path has */ +#define _WALK_URI_SZ 600 + +struct walk_frame { + DasNode* pParent; /* container searched at this level, NULL for a seed */ + const char* sRest; /* path below pParent, its separator stripped */ + bool bListing; /* loaded children exhausted, now in the listing */ + ptrdiff_t iLoaded; /* next loaded child to look at */ + const das_json_dict_el* pChild; /* listing entry in hand, NULL before the first */ + const char* sChild; /* its name, borrowed from the parent's document */ + const DasJdo* pUrls; /* its urls array, NULL for a seed */ + size_t iUrl; /* first candidate still on offer */ + size_t uUrls; /* candidates in all, string entries only */ + DasNode* pNode; /* loaded for candidate iUrl, not yet attached */ + char sUri[_WALK_URI_SZ]; /* the URI of the node this frame loads */ +}; + +struct das_node_walk { + char* sPath; /* owned copy, frames point into it */ + DasNode* pFrom; /* borrowed, NULL when loading a seed */ + const char** aSeedUrl; + size_t uSeedUrl; + struct walk_frame aFrame[_WALK_DEPTH]; + size_t uFrames; + das_walk_state_e state; + DasNode* pResult; + char sNear[_WALK_URI_SZ]; /* deepest container searched, "" if none */ + const char* sNearRest; + size_t uDeepest; +}; + +static void _DasNode_attach(DasNode* pParent, DasNode* pChild, const char* sName) +{ + DasCatNode* pCat = (DasCatNode*)pParent; + DasAry_append(pCat->pSubNodes, (const ubyte*)&pChild, 1); + DasAry_append(pCat->pSubPaths, (const ubyte*)sName, strlen(sName) + 1); + DasAry_markEnd(pCat->pSubPaths, DIM1); } -static DasNode* _DasNode_loadSubNode_spaseCat( - DasCatNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -){ - daslog_error("Spase catalogs are not yet supported"); +/* The name a loaded child was attached under, "" for a bad entry */ +static const char* _DasCatNode_loadedName(const DasCatNode* pCat, ptrdiff_t i, size_t* pLen) +{ + size_t uLen = 0; + const char* s = (const char*)DasAry_getBytesIn(pCat->pSubPaths, DIM1_AT(i), &uLen); + if((s == NULL)||(uLen == 0)){ *pLen = 0; return ""; } + *pLen = uLen - 1; /* the stored NUL is not part of the name */ + return s; +} + +static bool _DasCatNode_hasLoaded(const DasCatNode* pCat, const char* sName) +{ + ptrdiff_t n = DasAry_lengthIn(pCat->pSubNodes, DIM0); + for(ptrdiff_t i = 0; i < n; ++i){ + size_t uLen = 0; + const char* s = _DasCatNode_loadedName(pCat, i, &uLen); + if((uLen == strlen(sName)) && (strncmp(s, sName, uLen) == 0)) return true; + } + return false; +} + +/* Candidate number n of a frame, counting string entries only */ +static const char* _DasNodeWalk_url(const DasNodeWalk* pThis, const struct walk_frame* pF, size_t n) +{ + if(pF->pUrls == NULL) return (n < pThis->uSeedUrl) ? pThis->aSeedUrl[n] : NULL; + size_t k = 0; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pF->pUrls); pEl != NULL; pEl = pEl->next){ + const char* s = DasJdo_string(pEl->value); + if(s == NULL) continue; + if(k++ == n) return s; + } return NULL; } -DasNode* DasNode_subNode( - DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent -){ - if((sRelPath == NULL) || (sRelPath[0] == '\0')){ - das_error(DASERR_NODE, "Empty relative path, can't lookup scheme definition"); - return NULL; +static size_t _DasNode_countUrls(const DasNode* pParent, const char* sChild, const DasJdo* pUrls) +{ + size_t u = 0; + int i = 0; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pUrls); pEl != NULL; pEl = pEl->next, ++i){ + if(DasJdo_string(pEl->value) != NULL) ++u; + else daslog_error_v("From %s: %s/%s/%d element is not a string", + pParent->sURL, D2FRAG_URLS, sChild, i); } - if(!DasNode_isCatalog(pThis)){ - daslog_error_v("Node %s from %s is a terminating node", DasNode_label(pThis), - DasNode_srcUrl(pThis)); - return NULL; + return u; +} + +/* Remember the deepest container on the path, for DasNodeWalk_nearest(). + * On the path means the rest of the path continues below it with its + * separator, the same test _DasNode_subMatch() starts with; a look-alike + * such as "a" for "ab/x" is not on the path. Whether anything under it + * matches does not matter: "uiowa" is the nearest node for + * "uiowa/nope/deeper" even though it lists no "nope". */ +static void _DasNodeWalk_nearSet(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, const char* sRest) +{ + if(uDepth <= pThis->uDeepest) return; + pThis->uDeepest = uDepth; + strncpy(pThis->sNear, pNode->sPath, sizeof(pThis->sNear) - 1); + pThis->sNearRest = sRest; +} + +/* For a child met on the way: it is on the path only if the rest continues + * below it with its separator */ +static void _DasNodeWalk_near(DasNodeWalk* pThis, size_t uDepth, const DasNode* pNode, const char* sRem) +{ + const char* sSep = ((const DasCatNode*)pNode)->sPathSep; + size_t uSep = strlen(sSep); + if(uSep > 0){ + if(strncmp(sRem, sSep, uSep) != 0) return; + sRem += uSep; } - DasCatNode* pCat = (DasCatNode*)pThis; + _DasNodeWalk_nearSet(pThis, uDepth, pNode, sRem); +} - if((pCat->sPathSep[0] != '\0')&& - (strncmp(sRelPath, pCat->sPathSep, strlen(pCat->sPathSep)) == 0)){ - sRelPath += strlen(pCat->sPathSep); +static bool _DasNodeWalk_push(DasNodeWalk* pThis, DasNode* pParent, const char* sRest) +{ + if(pThis->uFrames == _WALK_DEPTH){ + daslog_error_v("Catalog path %s is more than %d levels deep", pThis->sPath, _WALK_DEPTH); + return false; } + struct walk_frame* pF = pThis->aFrame + pThis->uFrames; + memset(pF, 0, sizeof(*pF)); + pF->pParent = pParent; + + /* The rest may start with the separator this container uses, "/uiowa" + under site: or not, "uiowa"; both name the same child */ + const char* sSep = ((DasCatNode*)pParent)->sPathSep; + size_t uSep = strlen(sSep); + if((uSep > 0) && (strncmp(sRest, sSep, uSep) == 0)) sRest += uSep; + pF->sRest = sRest; + pThis->uFrames += 1; + _DasNodeWalk_nearSet(pThis, pThis->uFrames, pParent, sRest); /* already stripped */ + return true; +} - /* Search the local child cache first before calling on the inter-tubes */ - const char* sSubPath = NULL; - const char* sSubRelPath = NULL; - ptrdiff_t uChildren = DasAry_lengthIn(pCat->pSubNodes, DIM0); - size_t uLen = 0; - DasNode* pNode = NULL; - DasNode* pRet = NULL; - for(ptrdiff_t u = 0; u < uChildren; ++u){ - sSubPath = (const char*)DasAry_getBytesIn(pCat->pSubPaths, DIM1_AT(u), &uLen); - - /* From the DasAry's point of view, the item length includes the null - * because it has to store the null. But from strncmp's point of view - * we don't count the null because the rel path might not be a terminating - * path (i.e. it may have components below the child */ - if(uLen == 0){ - daslog_warn_v("Node at %s has a zero-length sub-path", pNode->sURL); - continue; - } - else{ - uLen -= 1; - } +/* Back out one level. The frame above drops the node it descended into + and moves to its next candidate; a frame that descended into an already + loaded child has nothing to drop. */ +static void _DasNodeWalk_pop(DasNodeWalk* pThis) +{ + struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + if(pF->pNode != NULL){ _DasNode_delTree(pF->pNode); pF->pNode = NULL; } + pThis->uFrames -= 1; + if(pThis->uFrames == 0) return; + pF = pThis->aFrame + (pThis->uFrames - 1); + if(pF->pNode != NULL){ + _DasNode_delTree(pF->pNode); + pF->pNode = NULL; + pF->iUrl += 1; + } +} - if(strncmp(sSubPath, sRelPath, uLen) == 0){ +static void _DasNodeWalk_finish(DasNodeWalk* pThis, DasNode* pResult) +{ + for(size_t k = 0; k < pThis->uFrames; ++k){ + struct walk_frame* pF = pThis->aFrame + k; + if(pF->pNode == NULL) continue; + if(pF->pParent != NULL) _DasNode_attach(pF->pParent, pF->pNode, pF->sChild); + pF->pNode = NULL; + } + if(pThis->pFrom == NULL) pResult->bOwner = true; /* a seed, the caller's */ + pThis->pResult = pResult; + pThis->state = das_walk_done; +} - /* This child completes at least part of the subpath */ - pNode = *((DasNode**) DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(u))); +/* Move to the next state: a fetch step, done, or failed */ +static void _DasNodeWalk_advance(DasNodeWalk* pThis) +{ + while(pThis->uFrames > 0){ + struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); - sSubRelPath = sRelPath + uLen; - if(sSubRelPath[0] == '\0'){ - /* actually it completes all of it*/ - return pNode; + /* Candidates still on offer for the child in hand */ + if((pF->pNode == NULL) && pF->bListing && (pF->iUrl < pF->uUrls)){ + pThis->state = das_walk_fetch; + return; + } + if(pF->pParent == NULL){ _DasNodeWalk_pop(pThis); continue; } /* every seed URL tried */ + + DasCatNode* pCat = (DasCatNode*)pF->pParent; + + /* Children already loaded, so a walked path costs no document */ + if(!pF->bListing){ + ptrdiff_t n = DasAry_lengthIn(pCat->pSubNodes, DIM0); + bool bPushed = false; + while(pF->iLoaded < n){ + ptrdiff_t i = pF->iLoaded++; + size_t uLen = 0; + const char* sName = _DasCatNode_loadedName(pCat, i, &uLen); + if(uLen == 0){ + daslog_warn_v("Node at %s has a zero-length sub-path", pF->pParent->sURL); + continue; + } + if(strncmp(sName, pF->sRest, uLen) != 0) continue; + DasNode* pKid = *((DasNode**)DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(i))); + const char* sRem = pF->sRest + uLen; + if(sRem[0] == '\0'){ _DasNodeWalk_finish(pThis, pKid); return; } + if(DasNode_isCatalog(pKid)) _DasNodeWalk_near(pThis, pThis->uFrames + 1, pKid, sRem); + if(!_DasNode_subMatch(pKid, sRem)) continue; + if(_DasNodeWalk_push(pThis, pKid, sRem)){ bPushed = true; break; } } - else{ - /* There's more, see if it's possible for this child to complete - * the path, if not don't fail, just keep looking */ - if( ! _checkSubMatch(pNode, sSubRelPath)) continue; + if(bPushed) continue; + pF->bListing = true; + pF->pChild = NULL; + } - pRet = DasNode_subNode(pNode, sSubRelPath, pMgr, sAgent); - if(pRet != NULL) return pRet; + /* The listing: the next child whose name starts the rest */ + const DasJdo* pDir = (const DasJdo*)pCat->pContainer; + const das_json_dict_el* pEl = NULL; + if((pDir != NULL) && (pDir->type == das_json_type_dict)) + pEl = (pF->pChild == NULL) ? DasJdo_dictFirst(pDir) : pF->pChild->next; + else + daslog_error_v("Catalog container node missing for node from %s", pF->pParent->sURL); + + bool bOffer = false; + for(; pEl != NULL; pEl = pEl->next){ + const char* sChild = pEl->name->string; + size_t uLen = strlen(sChild); + if(strncmp(sChild, pF->sRest, uLen) != 0) continue; + if(_DasCatNode_hasLoaded(pCat, sChild)) continue; + + /* Broken syntax: stop using this listing, the level above may + have another copy */ + const DasJdo* pStub = pEl->value; + if((pStub == NULL)||(pStub->type != das_json_type_dict)){ + daslog_error_v("Catalog error @ %s: sub item '%s' is not a dictionary", + pF->pParent->sURL, sChild); + break; } + const DasJdo* pUrls = DasJdo_get(pStub, D2FRAG_URLS); + if((pUrls == NULL)||(pUrls->type != das_json_type_ary)){ + daslog_error_v("From %s: %s element of node '%s' doesn't have a %s array", + pF->pParent->sURL, pCat->sContainer, sChild, D2FRAG_URLS); + break; + } + pF->pChild = pEl; + pF->sChild = sChild; + pF->pUrls = pUrls; + pF->iUrl = 0; + pF->uUrls = _DasNode_countUrls(pF->pParent, sChild, pUrls); + /* A root has no URI, so its children are named by bare name and + the levels below by their path from the root */ + if(pF->pParent->sPath[0] == '\0') + snprintf(pF->sUri, sizeof(pF->sUri), "%s", sChild); + else + snprintf(pF->sUri, sizeof(pF->sUri), "%s%s%s", pF->pParent->sPath, pCat->sPathSep, sChild); + if(pF->uUrls == 0) continue; + bOffer = true; + break; } + if(bOffer){ pThis->state = das_walk_fetch; return; } + + /* A miss at one level is normal while the walk backs out and tries + other candidates or children, so it is not an error here. The + driver knows whether the whole path failed and has the user's + wording of it. */ + const char* sLabel = DasNode_label(pF->pParent); + daslog_info_v("Node %s (URI '%s') has no child node that starts with %s", + sLabel ? sLabel : "", pF->pParent->sPath, pF->sRest); + _DasNodeWalk_pop(pThis); } + pThis->state = das_walk_failed; +} - /* I don't have this sub node: Internet! To the rescue! */ - - switch(pThis->nType){ - case d2node_collection: - case d2node_catalog: - case d2node_srcset: - return _DasNode_loadSubNode_dasCat(pCat, sRelPath, pMgr, sAgent); - case d2node_spdf_cat: - return _DasNode_loadSubNode_spdfCat(pCat, sRelPath, pMgr, sAgent); - case d2node_spase_cat: - return _DasNode_loadSubNode_spaseCat(pCat, sRelPath, pMgr, sAgent); - break; - default: - /* Should have been ruled out by constructor, but double check since - * new types may be added */ - das_error(DASERR_ASSERT, "Logic error in libdas2"); return NULL; +static DasNodeWalk* _dasnodewalk_alloc(DasNode* pFrom, const char* sRelPath) +{ + DasNodeWalk* pThis = (DasNodeWalk*)calloc(1, sizeof(DasNodeWalk)); + if(pThis == NULL){ + das_error(DASERR_NODE, "Couldn't allocate a catalog walk"); + return NULL; } - - return NULL; + pThis->pFrom = pFrom; + + /* An owned copy even when empty, das_strdup() has nothing to give for "" */ + size_t uPath = (sRelPath != NULL) ? strlen(sRelPath) : 0; + pThis->sPath = (char*)calloc(uPath + 1, 1); + if(pThis->sPath == NULL){ + das_error(DASERR_NODE, "Couldn't allocate a catalog walk"); + free(pThis); + return NULL; + } + if(uPath > 0) memcpy(pThis->sPath, sRelPath, uPath); + return pThis; } -/* ************************************************************************* */ -/* Selective Destruction */ - -/* only deletes non-root nodes and any contained non-root nodes. Not public: - a non-root node lives in its parent's pSubNodes cache, freeing one alone - would leave a dangling entry there. Callers only ever own roots. */ -static void del_NonRootNode(DasNode* pThis) +DasNodeWalk* new_DasNodeWalk(DasNode* pFrom, const char* sRelPath) { - if(pThis == NULL) return; - if(pThis->bIsRoot) return; - - /* Cascade deletes for non-root sub items */ - if(DasNode_isCatalog(pThis)){ - DasCatNode* pCat = (DasCatNode*)pThis; + if(pFrom == NULL){ + daslog_error("A walk needs a node to start from; for the compiled in seed " + "see new_DasNodeWalk_seed()"); + return NULL; + } + if(!DasNode_isCatalog(pFrom)){ + daslog_error_v("Node %s from %s is a terminating node", + DasNode_label(pFrom), DasNode_srcUrl(pFrom)); + return NULL; + } + DasNodeWalk* pThis = _dasnodewalk_alloc(pFrom, sRelPath); + if(pThis == NULL) return NULL; - ptrdiff_t uNodes = DasAry_lengthIn(pCat->pSubNodes, DIM0); - for(ptrdiff_t u = 0; u < uNodes; ++u){ - DasNode** ppSub = (DasNode**)DasAry_getAt(pCat->pSubNodes, vtUnknown, IDX0(u)); - del_NonRootNode(*ppSub); - } - dec_DasAry(pCat->pSubNodes); - dec_DasAry(pCat->pSubPaths); + if(pThis->sPath[0] == '\0'){ + pThis->pResult = pFrom; + pThis->state = das_walk_done; + return pThis; } - /* Now del self */ - if(!DasNode_isJson(pThis)) - das_error(DASERR_NODE, "Memory Leak! Handling for non-JSON nodes has " - "not been implemented"); - else - free(pThis->pDom); + _DasNodeWalk_push(pThis, pFrom, pThis->sPath); /* cannot be too deep at 0 */ + _DasNodeWalk_advance(pThis); + return pThis; +} - free(pThis); +DasNodeWalk* new_DasNodeWalk_seed(void) +{ + DasNodeWalk* pThis = _dasnodewalk_alloc(NULL, NULL); + if(pThis == NULL) return NULL; + pThis->aSeedUrl = das_seed_urls(&pThis->uSeedUrl); + + /* One frame with no parent: its candidates are the seed URLs and the + document that arrives is the result */ + struct walk_frame* pF = pThis->aFrame; + pF->sRest = pThis->sPath; + pF->bListing = true; + pF->uUrls = pThis->uSeedUrl; + pThis->uFrames = 1; + _DasNodeWalk_advance(pThis); + return pThis; } -/* only deletes root nodes and any contained non-root nodes */ -void del_RootNode(DasNode* pThis) + +void del_DasNodeWalk(DasNodeWalk* pThis) { if(pThis == NULL) return; - if(!pThis->bIsRoot) return; + /* Frames hold unattached nodes only while the walk is under way; each + is a separate tree since the next frame's node is not attached to + it, so order does not matter */ + for(size_t k = 0; k < pThis->uFrames; ++k) + if(pThis->aFrame[k].pNode != NULL) _DasNode_delTree(pThis->aFrame[k].pNode); + free(pThis->sPath); + free(pThis); +} - pThis->bIsRoot = false; - del_NonRootNode(pThis); +das_walk_state_e DasNodeWalk_state(const DasNodeWalk* pThis){ return pThis->state; } + +size_t DasNodeWalk_urlCount(const DasNodeWalk* pThis) +{ + if(pThis->state != das_walk_fetch) return 0; + const struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + return pF->uUrls - pF->iUrl; } -/* ************************************************************************* */ -/* Root Node Construction */ +const char* DasNodeWalk_url(const DasNodeWalk* pThis, size_t uIdx) +{ + if(uIdx >= DasNodeWalk_urlCount(pThis)) return NULL; + const struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + return _DasNodeWalk_url(pThis, pF, pF->iUrl + uIdx); +} -DasNode* new_RootNode_url( - const char* sUrl, const char* sPathUri, DasCredMngr* pMgr, const char* sAgent -){ - /* Since they want a specific URL, go for the max timeout */ - DasNode* pNode = _DasNode_mkNode(sUrl, sPathUri, pMgr, sAgent, DASHTTP_TO_MAX); - if(pNode) pNode->bIsRoot = true; - return pNode; +const char* DasNodeWalk_uri(const DasNodeWalk* pThis) +{ + if(pThis->state != das_walk_fetch) return ""; + return pThis->aFrame[pThis->uFrames - 1].sUri; } -DasNode* new_RootNode( - const char* sPathUri, DasCredMngr* pMgr, const char* sAgent +DasErrCode DasNodeWalk_feed( + DasNodeWalk* pThis, size_t uIdx, const char* pBytes, size_t uBytes, + const char* sFinalUrl ){ - /* Find the node using the path, start at the root */ - DasNode* pTop = NULL; - DasNode* pNode = NULL; - - /* Changing node lookup and time outs. New strategy, try to contact a node - if it doesn't work within the time out, try the next one. If all fail - then increase the time out and try again. - */ - - float rConSec = DASHTTP_TO_MIN; - - while((rConSec <= DASHTTP_TO_MAX )&&(pNode == NULL)){ - for(int i = 0; i < g_nDasDefRoots; ++i){ - /* Credentials should never be needed for top level nodes */ - pTop = _DasNode_mkNode(g_lDasDefRoots[i], NULL, NULL, sAgent, rConSec); - if(pTop == NULL) continue; - if(pTop->nType != d2node_catalog) continue; - - if(sPathUri == NULL){ /* They may be looking for a root node */ - pNode = pTop; - break; - } - - /* Depends on the fact that top node is a das catalog */ - pNode = _DasNode_loadSubNode_dasCat((DasCatNode*)pTop, sPathUri, pMgr, sAgent); - if(pNode != NULL) break; - } - rConSec *= DASHTTP_TO_MULTI; + /* Misuse by the driver, logged rather than raised: a host that mixes up + its candidates should not take the program down with it */ + if(pThis->state != das_walk_fetch){ + daslog_error("The walk is not waiting for a document"); + return DASERR_NODE; } + size_t uOffer = DasNodeWalk_urlCount(pThis); + if(uIdx >= uOffer){ + daslog_error_v("Candidate %zu is past the %zu on offer", uIdx, uOffer); + return DASERR_NODE; + } + + struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + const char* sUrl = _DasNodeWalk_url(pThis, pF, pF->iUrl + uIdx); + if(sFinalUrl == NULL) sFinalUrl = sUrl; + DasNode* pNode = _dasnode_fromJson(pBytes, uBytes, sUrl, sFinalUrl, pF->sUri); + if((pNode != NULL) && (pF->pParent == NULL) && (pNode->nType != d2node_catalog)){ + daslog_info_v("%s is not a Catalog node, skipping it as a seed", sUrl); + _DasNode_delTree(pNode); + pNode = NULL; + } if(pNode == NULL){ - daslog_error_v("Failed to load the requested node %s from any of the " - "built in catalog trees", sPathUri); - return NULL; + pF->iUrl += uIdx + 1; + _DasNodeWalk_advance(pThis); + return DASERR_NODE; } + pF->iUrl += uIdx; /* the one in hand; a back out moves past it */ + pF->pNode = pNode; - /* I'm keeping this node, so set it as a root item so that it will not be - * deleted without an explicit call to del_RootNode, then delete the rest, - * pTop will not be deleted if pTop == pNode */ - pNode->bIsRoot = true; - del_NonRootNode(pTop); + const char* sRem = (pF->pParent == NULL) ? pF->sRest : pF->sRest + strlen(pF->sChild); + if(sRem[0] == '\0'){ _DasNodeWalk_finish(pThis, pNode); return DAS_OKAY; } - return pNode; + if(DasNode_isCatalog(pNode)) _DasNodeWalk_near(pThis, pThis->uFrames + 1, pNode, sRem); + if(DasNode_isCatalog(pNode) && _DasNode_subMatch(pNode, sRem) && _DasNodeWalk_push(pThis, pNode, sRem)){ + _DasNodeWalk_advance(pThis); + return DAS_OKAY; + } + + /* A good document that leads nowhere: a look-alike child, or a mirror + whose listing is behind. Not the driver's problem. */ + daslog_debug_v("Nothing under %s continues the path %s", sUrl, sRem); + _DasNode_delTree(pNode); + pF->pNode = NULL; + pF->iUrl += 1; + _DasNodeWalk_advance(pThis); + return DAS_OKAY; } +void DasNodeWalk_fail(DasNodeWalk* pThis, size_t uIdx, const char* sWhy) +{ + if(uIdx >= DasNodeWalk_urlCount(pThis)) return; + struct walk_frame* pF = pThis->aFrame + (pThis->uFrames - 1); + daslog_debug_v("%s: %s", _DasNodeWalk_url(pThis, pF, pF->iUrl + uIdx), sWhy ? sWhy : "not available"); + pF->iUrl += uIdx + 1; + _DasNodeWalk_advance(pThis); +} + +DasNode* DasNodeWalk_result(const DasNodeWalk* pThis) +{ + return (pThis->state == das_walk_done) ? pThis->pResult : NULL; +} + +const char* DasNodeWalk_nearest(const DasNodeWalk* pThis, const char** psRest) +{ + if(psRest != NULL) *psRest = NULL; + if((pThis->state != das_walk_failed)||(pThis->uDeepest == 0)) return NULL; + if(psRest != NULL) *psRest = pThis->sNearRest; + return pThis->sNear; +} diff --git a/das3/node.h b/das3/node.h index c3d7484..1da4bb0 100644 --- a/das3/node.h +++ b/das3/node.h @@ -28,107 +28,60 @@ extern "C" { -/** @defgroup catalog Catalogs - * Provides utilities for reading remote data source catalogs and caching their - * contents in RAM. - * - * There are two basic methods of catalog object acquisition, finding a node by - * it's path from the global root of the federated das2 catalog system: - * (ex: tag:das2.org,2012:site:/swri/mars_express/aspera3/sciels) - * or by an loading an explicit URL - * (ex: http://das2.org/catalog/das/site/uiowa/cassini/rpws/survey.json). - * - * When describing the das2 catalog C-API there are two meanings for the word - * Root. - * - * 1) Memory Roots - A node can be the in-memory root of a linked list of nodes. - * - * 2) Global Root - Root can also refer to the global das2 top-level un-named - * node which resides at http://das2.org/catalog/index.json and it's mirrors. - * - * Any node loaded using new_RootNode() or new_RootNode_url() is an *in memory* - * root node. It might not be the root of a federated catalog system, but it - * is still a top level memory item that manages a memory cache for it's - * subnodes. If new_RootNode() happens to be called with a URI of 'NULL' then - * the in-memory root corresponds to the federated catalog root. The reason for - * splitting the concept of the in-memory root from the global catalog root - * is to provide for isolated catalogs that are not part of the global system. - * - * When you instantiate a node from a random URL you have to give it a name. - * Just like files on a disk, catalog nodes do not know their name, it is - * derived from the name provided by the higher level catalog entries (up to - * root, which has no name). - * - * An Example of getting a data source interface definition follows: - * @code{.cpp} - - // Get the site root and safely cast the output. Any particular point in - // the catalog can be a root node for the purposes of sub-item queries and - // memory management. By calling new_RootNode() the returned item is a local - // root object for memory management purposes. The higher level nodes that - // were traversed to get to the requested item are deleted from memory. - - DasNode* pRoot = new_RootNode("tag:das2.org,2012:site:/uiowa", NULL, NULL); - if(!DasNode_isCatalog(pNode)) handle_error(); - - // Use the site catalog to get the Cassini Saturn SLS2 data source description - - const char* sRelPath = "uiowa/cassini/ephemeris/saturn_sls2"; - DasNode* pNode = DasNode_subNode(pRoot, sRelPath, NULL, NULL); - - if(!(DasNode_isStreamSource(pNode))) handle_error(); - - // Get the HTTP get Query interface definition - - const DasJdo* pDef = DasNode_getJdo(pNode, "SOURCE/QUERY_PARAMS"); - if(pDef == NULL) handle_error(); - - // Freeing the root node will delete all lower nodes - - del_DasNode(pRoot); - - @endcode - * - * You now have a JSON object providing the query definition see - * http://das2.org/datasource.html for more information on the content of das2 - * data source definitions. - * - * For isolated sites, or for testing purposes catalog nodes can be loaded - * directly. The following example acquires a local test root and then a - * specific test node. - * - @code - - // Assume this site uses a password since it is a testing area. Create a - // credentials manager and pass a basic auth hash so we aren't bothered during - // testing. - - DasCredMngr* pMngr = new_CredMngr(NULL); - das_credential cred; - das_cred_init(&cred, "my.domain.com", "Regression Test Catalog", NULL, - "VVNFUk5BTUU6UEFTU1dPUkQ="); - CredMngr_addCred(pMngr, &cred); - - const char* sTestUrl = "http://my.domain.com/das2/regtest.json"; - DasNode* pRoot = new_RootNode_url(sTestUrl, pMngr, NULL); - if(!DasNode_isCatalog(pNode)) handle_error(); - - // At this point sub-items can be loaded from the local root as above. - - DasNode* pNode = DasNode_subNode("MarsExpress/aspera/els", pMngr, NULL); - +/** @addtogroup catalog + * + * @par Nodes without a session + * + * A DasNode is one parsed catalog document: its type, URI, source URL, + * and the document itself, with its children listed by name and type and + * the URLs each would load from. Nothing in this header fetches. A node + * is built from bytes the caller obtained, see new_DasNode_json(), and a + * path below it is resolved by a DasNodeWalk, which says which document + * it needs next and takes what the caller hands back. Programs on a real + * machine do not drive the walk themselves: a DasCat session, see + * catalog.h, reads files, keeps a cache, makes the HTTP requests and + * loops the walk until it stops. A browser build drives it from + * JavaScript with fetch(). The compiled in seed, the index of the das2 + * catalog, is loaded through the same walk interface by + * new_DasNodeWalk_seed(). + * + * Ownership follows general das2C rules: the node a constructor hands + * you is yours, and del_DasNode() on it frees the whole tree loaded below + * it. Children a walk attaches are managed by their parent and go with + * it; del_DasNode() ignores them. + * + * Catalog nodes do not know their names. Like files on a disk, a + * node's URI comes from the entries above it, up to the root, which has + * none. So a node built from bytes is named by the caller, or not at + * all: a seed given no prefix is a root, and the nodes below it are named + * by their path from it. + * + * Driving a walk by hand, which is what a browser build does: + * + * @code + DasNode* pSite = new_DasNode_json(sBytes, uLen, sUrl, "tag:das2.org,2012:site:/uiowa"); + + DasNodeWalk* pWalk = new_DasNodeWalk(pSite, "juno/wav/survey"); + while(DasNodeWalk_state(pWalk) == das_walk_fetch){ + const char* sNext = DasNodeWalk_url(pWalk, 0); + if(my_fetch(sNext, &sDoc, &uDoc)) + DasNodeWalk_feed(pWalk, 0, sDoc, uDoc, NULL); + else + DasNodeWalk_fail(pWalk, 0, "no answer"); + } + DasNode* pSurvey = DasNodeWalk_result(pWalk); // NULL if the walk failed + del_DasNodeWalk(pWalk); // pSurvey lives on under pSite @endcode - * - * */ /* Defines for common path_uri's */ #define D2URI_ROOT "tag:das2.org,2012:" #define D2URI_SITES D2URI_ROOT "site" #define D2URI_SITE_UIOWA D2URI_SITES ":/uiowa" - + /* Defines for common document fragments */ #define D2FRAG_TYPE "type" +#define D2FRAG_VERSION "version" #define D2FRAG_NAME "name" #define D2FRAG_LABEL "label" #define D2FRAG_TITLE "title" @@ -145,7 +98,7 @@ extern "C" { #define D2CV_TYPE_STREAM "HttpStreamSrc" #define D2CV_TYPE_TIMEAGG "FileTimeAgg" #define D2CV_TYPE_SPASE "SpaseRecord" -#define D2Cv_TYPE_SPDF_MASTER "SpdfMasterCat" +#define D2CV_TYPE_SPDF_MASTER "SpdfMasterCat" /** Catalog node type */ typedef enum das_node_type_enum { @@ -158,144 +111,49 @@ typedef enum das_node_type_enum { * @{ */ -/** Base type for das2 catalog nodes. */ -typedef struct das_node { - das_node_type_e nType; /* Holds the node type */ - char sURL[512]; /* Holds source URL for this node */ - char sPath[512]; /* Holds the Path URI for this node */ - bool bIsRoot; /* True if a local root node (i.e. is a memory manager) */ - void* pDom; /* A pointer to the document tokens. */ -} DasNode; - -/** @} */ - -/** Return the compiled in catalog root URLs - * @param pLen - A pointer to a size_t to received the number of builtin roots - * @return A pointer to an array of null terminated strings. +/** A das catalog node, opaque. + * + * Nodes are made by new_DasNode_json() and by a DasNodeWalk, read + * through the accessors below, and freed by del_DasNode() on a tree top, + * which takes the nodes walked to below it along. */ -DAS_API const char** das_root_urls(size_t* pLen); +typedef struct das_node DasNode; -/** Create a new root catalog node via a path URI - * - * Get a catalog node that is not attached to any parent nodes. If the - * node acquired is a container type such as Root, Scheme, or Catalog then - * it can be used to acquire further nodes. - * - * This function consults the distributed das2 catalog to find and load nodes. - * See new_RootNode_url() for a version that only loads a specified URL. - * - * @code{.cpp} - - DasNode* pRoot = NULL; - - // Get the das2 site root catalog node - pRoot = new_RootNode(D2URI_DAS_SITE_ROOT, NULL, NULL, NULL); - - // Get the U. Iowa site catalog top node - pRoot = new_RootNode(D2URI_DAS_SITE_ROOT "/uiowa", NULL, NULL); - - @endcode - * - * @param sPathUri Retrieve a catalog node given a global path URI. This a - * location from the top of the federated catalog system, not - * a file system or web path. - * - * @param sAgent The user agent string you wish to send to the server. If - * NULL then the string "libdas/3" is sent. - * - * @param pMgr A credentials manager object to consult if a password is - * requested. May be set to NULL to indicate that only public - * items may be requested. Usually this is not needed as most - * catalog entries are public though the data sources they - * describe may not be. - * - * @return A new DasNode object allocated on the heap, or NULL if object - * resolution failed. Calling del_RootNode() for the returned - * pointer will delete all sub-nodes from memory as well. - * - * @memberof DasNode - */ -DAS_API DasNode* new_RootNode( - const char* sPathUri, DasCredMngr* pMgr, const char* sAgent -); +/** @} */ -/** Create a new root catalog node via direct URL - * - * Get a catalog node that is not attached to any parent nodes. If the - * node acquired is a container type such as Root, Scheme, or Catalog then - * it can be used to acquire further nodes. - * - * There are two basic methods of node acquisition, set a path URI and let the - * library find the node using the built-in global catalog location, or - * provide an explicit URL. - * - * @code{.cpp} - - DasNode* pRoot = NULL; - - // Get a standalone local Voyager catalog root - pRoot = new_RootNode(NULL, "http://my.site.gov/test/local_vgr.json", NULL, NULL); - - // Get a standalone data source description for Voyager PLS data - pRoot = new_RootNode(NULL, "http://my.site.gov/test/local_vgr/pls.json", NULL, NULL); - - @endcode - * - * @param sUrl Retrieve a catalog node from an explicit Url. No searches - * will be preformed. - * - * @param sPathUri Since the node was not loaded by following the federated - * catalog system to an end point, you will have to tell the node - * it's name. Any name will worked and it's saved internally. - * - * @param sAgent The user agent string you wish to send to the server. If - * NULL then the string "libdas/3" is sent. - * - * @param pMgr A credentials manager object to consult if a password is - * requested. May be set to NULL to indicate that only public - * items may be requested. Usually this is not needed as most - * catalog entries are public though the data sources they - * describe may not be. - * - * @return A new DasNode object allocated on the heap, or NULL if object - * resolution failed. Use DasNode_type() to determine the which - * type of Catalog item was acquired. - * - * @memberof DasNode +/** The compiled in seed URLs: the index of the das catalog and its mirrors + * + * The seed is a root, a node with no name, whose children are the top + * level catalogs, each named by a whole URI prefix such as + * "tag:das2.org,2012:". new_DasNodeWalk_seed() loads it from whichever + * of these answers, in order. + * + * @param pLen Receives the number of URLs. May be NULL. + * @return An array of NUL terminated strings, static. */ -DAS_API DasNode* new_RootNode_url(const char* sUrl, const char* sPathUri, - DasCredMngr* pMgr, const char* sAgent); +DAS_API const char** das_seed_urls(size_t* pLen); -/** Get a das2 catalog node contained item. - * - * This function will search down the catalog hierarchy and find the requested - * node, downloading any intermediate nodes as needed. If the descendant node - * has already been downloaded no new network activity is generated. - * - * @param pThis A pointer to a node of type D2C_CATALOG - * - * @param sRelPath the ID of the descendant node. This id is the portion - * that would be appended to this node's PathUri to make the - * complete URI of the final node. sRelPath does not need to start - * with the sub-item separator (if one is defined for this catalog) - * but may at the callers option. For example, both: - * ":/uiowa" and "uiowa" work for getting the uiowa subnode from - * the site catalog, but "/uiowa" would not since the separator - * defined for site is ":/". - * - * @param sAgent The user agent string you wish to send to the server. If - * NULL then the string "libdas/3" is sent. - * - * @param pMgr A credentials manager object to consult if a password is - * requested. May be set to NULL to indicate that only public - * items may be requested. +/** Build a node from the JSON text of a node document * - * @return A pointer to the child node, or NULL if no such child existed - * in the node cache nor could not be downloaded. + * For programs that fetch documents themselves, a browser build in + * particular. Nothing is fetched and no file is read. The result is a + * tree top the caller owns, with no parent in memory; a container built + * this way lists its children, see DasNode_subCount(), and names the + * URLs each would load from, see DasNode_subStub(). + * + * @param pBytes The document text, need not be NUL terminated. + * @param uBytes The number of bytes in pBytes. + * @param sUrl Where the text came from, recorded as the node's source + * URL, see DasNode_srcUrl(). May be NULL. + * @param sPathUri The catalog URI of the node, shorthands expanded. NULL + * or empty makes it a root: no URI, and the nodes walked to + * below it are named by their path from it. + * @return A new node, to be freed with del_DasNode(), or NULL after + * logging a parse problem or an unrecognized node type. * @memberof DasNode */ -DAS_API DasNode* DasNode_subNode( - DasNode* pThis, const char* sRelPath, DasCredMngr* pMgr, const char* sAgent +DAS_API DasNode* new_DasNode_json( + const char* pBytes, size_t uBytes, const char* sUrl, const char* sPathUri ); /** Returns true this node can contain sub nodes. @@ -319,27 +177,33 @@ DAS_API bool DasNode_isSpaseRec(const DasNode* pNode); DAS_API bool DasNode_isSpdfCat(const DasNode* pNode); /** Get the path URI for this catalog node - * @param pThis the catalog node - * @return A pointer to the path URI string. All catalog nodes have a path URI + * + * Names are composed walking down: a seed given a prefix hands it to its + * children with its separator between, and so on. A seed given no + * prefix is a root, and a root is the one node with no URI; the compiled + * in das catalog index is the usual example. + * + * @param pThis the catalog node + * @return The URI, or NULL for a root * @memberof DasNode */ DAS_API const char* DasNode_pathUri(const DasNode* pThis); -/** Get the location from which this catalog node was read. - * @param pThis the catalog node - * @return A pointer to the source URL string. All catalog nodes are loaded - * from somewhere, das2C does not provide functions to generate - * node objects programmatically. +/** Get the location from which this catalog node was read + * @param pThis the catalog node + * @return The source URL, or "" for a node built from bytes with no + * location given, see new_DasNode_json(). Never NULL. * @memberof DasNode */ DAS_API const char* DasNode_srcUrl(const DasNode* pThis); /** Get the type of node - * This is a more specific question than the 'is' functions below, which should - * probably be used instead so that application code can work in a "duck-typing" - * manner. - * @param pThis the catalog node - * @return The node type from the + * + * A more specific question than the 'is' functions above, which are + * usually the better choice so that application code works in a duck + * typing manner. + * @param pThis the catalog node + * @return The node type, see das_node_type_e * @memberof DasNode */ DAS_API das_node_type_e DasNode_type(const DasNode* pThis); @@ -367,46 +231,282 @@ DAS_API const char* DasNode_title(const DasNode* pThis); */ DAS_API const char* DasNode_summary(const DasNode* pThis); +/** Get the node's "version" field, if provided + * + * This is the version of the node *definition* (ex: "0.5" for catalogs, + * "0.6" or "0.7" for HttpStreamSrc) not a data version. Clients need it + * because the interface layout of source nodes changed between 0.6 and 0.7. + * @returns the version string or NULL + * @memberof DasNode + */ +DAS_API const char* DasNode_version(const DasNode* pThis); + +/** Get a short label for the node, the "label" field if present, else "name" + * @returns the label, or NULL if the node has neither field + * @memberof DasNode + */ +DAS_API const char* DasNode_label(const DasNode* pThis); + +/** Get the separator a container node puts between its path URI and + * the name of a child. + * + * @returns "/" by default, the empty string if the node's "separator" is + * null, or NULL if this is not a container node. + * @memberof DasNode + */ +DAS_API const char* DasNode_pathSep(const DasNode* pThis); + +/** Number of children listed by a container node. + * + * This inspects the listing only, nothing is downloaded. Children of a + * container are known by name and type before they are loaded, just like + * directory entries. + * + * @returns the number of listed children, 0 for non-container nodes + * @memberof DasNode + */ +DAS_API size_t DasNode_subCount(const DasNode* pThis); + +/** Get the name of a listed child by index. No network activity. + * @param uIdx 0 to DasNode_subCount() - 1 + * @returns the child's name, or NULL if the index is out of range + * @memberof DasNode + */ +DAS_API const char* DasNode_subName(const DasNode* pThis, size_t uIdx); + +/** Get the listing entry (stub) for a child by index. No network activity. + * + * The stub is the dictionary the container holds for the child. It always + * has "type" and "urls", and often "label", "title" or "description". It + * is not the child node. A walk, or a session's DasCat_subNode(), loads that. + * + * @returns the stub dictionary, or NULL if the index is out of range + * @memberof DasNode + */ +DAS_API const DasJdo* DasNode_subStub(const DasNode* pThis, size_t uIdx); + +/** Get the node type of a listed child by index. No network activity. + * @returns the type, d2node_inv if the type string is unknown to this + * library (ex: "Das2DSDF", "WebSocSrc") or the index is bad + * @memberof DasNode + */ +DAS_API das_node_type_e DasNode_subType(const DasNode* pThis, size_t uIdx); + /** If true the internal data for this node are in JSON format - * - * @param pThis The catalog node - * @return True if DasNode_getJdo() usable + * + * Always true today; the hook is for catalogs in another format. + * @param pThis The catalog node + * @return True if DasNode_getJdo() is usable + * @memberof DasNode */ DAS_API bool DasNode_isJson(const DasNode* pThis); -/** Get a JSON document object at a fragment location in a node. +/** Get a JSON document object at a fragment location in a node. * @param pThis The catalog node - * @param sFragment The fragment path, for example "CONTACTS/TECH/0/EMAIL" - * @return The json object at that location, or NULL if the node does not + * @param sFragment The fragment path, for example "contacts/0/email", or + * NULL for the whole document + * @return The json object at that location, or NULL if the node does not * contain the given element. * @memberof DasNode */ DAS_API const DasJdo* DasNode_getJdo(const DasNode* pThis, const char* sFragment); -/** Get a JSON document object of a particular type at a fragment location - * @param pThis The catalog node - * @param type The type of object expected - * @param sFragment The fragment path, for example "CONTACTS/TECH/0/EMAIL" - * @return The json object at that location, or NULL if the node does not - * contain the given element, or if the given element is of the - * wrong type - * @memberof DasNode + +/* ************************************************************************* */ +/* Walking */ + +/** @addtogroup catalog + * @{ + */ + +/** A walk from a node to a descendant, opaque. + * + * The catalog's path resolution with the transport taken out. A walk + * knows which document it needs next and what to make of the one it is + * handed; whoever drives it reads files or makes HTTP requests. + * + * Each step offers the candidate URLs for one node, the mirrors a catalog + * lists for a child, in catalog order. A sequential driver takes index + * 0, and after a DasNodeWalk_fail() or a document that does not parse the + * list shrinks, so index 0 is always the next thing to try. A driver that + * fetches mirrors in parallel feeds whichever answered. A document that + * parses but leads nowhere, a look-alike child or a mirror out of date, + * is dropped silently and the next candidate offered. When a listing is + * exhausted the walk backs out a level and tries the level above's next + * candidate, so a dead mirror doesn't hide a live one. + * + * Nothing joins the tree until the walk succeeds; then the nodes it + * loaded are attached under their parents and a second walk of the same + * path needs no document at all. A walk that fails or is deleted early + * frees what it loaded. + * + * The driver's loop, for a program that fetches for itself: + * + * @code + DasNodeWalk* pWalk = new_DasNodeWalk(pSite, "juno/wav/survey"); + if(pWalk == NULL) handle_error(); + + while(DasNodeWalk_state(pWalk) == das_walk_fetch){ + + // Index 0 is always the next candidate to try, see below + const char* sUrl = DasNodeWalk_url(pWalk, 0); + + char* pDoc = NULL; + size_t uDoc = 0; + if(my_fetch(sUrl, &pDoc, &uDoc)){ + // DASERR_NODE here means the bytes were not a node; the walk has + // already moved on, so there is nothing to do but note it + if(DasNodeWalk_feed(pWalk, 0, pDoc, uDoc, NULL) != DAS_OKAY) + log("%s did not parse", sUrl); + free(pDoc); + } + else{ + DasNodeWalk_fail(pWalk, 0, "no answer"); + } + } + + DasNode* pSurvey = DasNodeWalk_result(pWalk); + if(pSurvey == NULL){ + const char* sRest = NULL; + const char* sNear = DasNodeWalk_nearest(pWalk, &sRest); + log("nothing at %s below %s", sRest, sNear); + } + del_DasNodeWalk(pWalk); // pSurvey lives on under pSite + @endcode + * + * The same loop loads the compiled in seed, new_DasNodeWalk_seed(): the + * candidates are then the seed URLs, see das_seed_urls(), and the result + * is the root of the das catalog, a seed the caller owns and frees with + * del_DasNode(). A lookup by full URI is a walk from it, and what it + * loads stays under it, so a client keeps the seed for its lifetime and + * the tree below it is its memory of the catalog. In a browser build the + * loop is the same again with an await on fetch() in place of my_fetch(). + */ +typedef struct das_node_walk DasNodeWalk; + +/** What a walk is waiting for */ +typedef enum das_walk_state_enum { + das_walk_fetch = 0, /**< a document, see DasNodeWalk_url() */ + das_walk_done, /**< nothing, DasNodeWalk_result() has the node */ + das_walk_failed /**< nothing, no path led to it, see DasNodeWalk_nearest() */ +} das_walk_state_e; + +/** Start a walk. + * + * @param pFrom The container to start from, borrowed; it must outlive + * the walk. + * + * @param sRelPath The path below pFrom, with or without the separator + * pFrom uses in front. Copied. Empty means pFrom itself and the + * walk is done at once. Not expanded: a full URI is spelled out, + * "tag:das2.org,2012:..." + * + * @returns A new walk, to be freed with del_DasNodeWalk(), already + * advanced to its first state. NULL after logging why not: no + * start node, or one that is not a container. + * @memberof DasNodeWalk + */ +DAS_API DasNodeWalk* new_DasNodeWalk(DasNode* pFrom, const char* sRelPath); + +/** Start a walk that loads the compiled in seed. + * + * One step, whose candidates are the URLs from das_seed_urls() in order, + * so a mirror that does not answer is passed over for the next. The + * result is the index of the das catalog, a root: it has no URI, its + * children are the top level catalogs by URI prefix, and it is the + * caller's to keep and to free with del_DasNode(). Everything else in + * the federated catalog is a walk from it. + * + * @returns A new walk, to be freed with del_DasNodeWalk(), waiting for + * a document. + * @memberof DasNodeWalk */ -DAS_API const DasJdo* DasNode_getJdoType( - const DasNode* pThis, enum das_json_type_e, const char* sFragment +DAS_API DasNodeWalk* new_DasNodeWalk_seed(void); + +/** Free a walk. A finished walk's result stays where the walk attached + * it, or in the caller's hands for a seed. Nodes loaded by a walk that + * did not finish go with it. + * @memberof DasNodeWalk */ +DAS_API void del_DasNodeWalk(DasNodeWalk* pThis); + +/** @memberof DasNodeWalk */ +DAS_API das_walk_state_e DasNodeWalk_state(const DasNodeWalk* pThis); + +/** The number of candidate URLs on offer, 0 unless the state is + * das_walk_fetch. @memberof DasNodeWalk */ +DAS_API size_t DasNodeWalk_urlCount(const DasNodeWalk* pThis); + +/** Candidate number uIdx, borrowed and valid until the next feed or fail, + * or NULL past the end. @memberof DasNodeWalk */ +DAS_API const char* DasNodeWalk_url(const DasNodeWalk* pThis, size_t uIdx); + +/** The catalog URI of the node this step would load, for messages and + * for a driver's cache. "" when loading a seed, which has none. + * @memberof DasNodeWalk */ +DAS_API const char* DasNodeWalk_uri(const DasNodeWalk* pThis); + +/** Hand the walk the document a candidate URL produced. + * + * @param uIdx Which candidate, see DasNodeWalk_url(). Candidates before + * it are dropped as not tried. + * @param pBytes The document text, need not be NUL terminated. + * @param uBytes Its length. + * @param sFinalUrl The URL after redirects, for messages. NULL means + * the candidate itself. + * @returns DAS_OKAY when the text parsed as a node, whether or not it led + * anywhere, and the walk has advanced. DASERR_NODE when it did + * not parse, or is not a Catalog when a seed was wanted, after + * logging; that candidate is dropped and the walk has advanced as + * after DasNodeWalk_fail(). Also DASERR_NODE, with nothing + * changed, when the walk was not waiting or uIdx is out of range. + * @memberof DasNodeWalk + */ +DAS_API DasErrCode DasNodeWalk_feed( + DasNodeWalk* pThis, size_t uIdx, const char* pBytes, size_t uBytes, + const char* sFinalUrl ); -/** Delete a root node freeing it's memory. - * All sub-nodes will be deleted as well. +/** Tell the walk a candidate URL produced nothing. + * @param uIdx Which candidate. It and those before it are dropped. + * @param sWhy For the debug log, may be NULL. + * @memberof DasNodeWalk */ +DAS_API void DasNodeWalk_fail(DasNodeWalk* pThis, size_t uIdx, const char* sWhy); + +/** The node a finished walk reached, or NULL. Attached under the start + * node's tree, or for new_DasNodeWalk_seed() the seed itself, the + * caller's to del_DasNode(). + * @memberof DasNodeWalk */ +DAS_API DasNode* DasNodeWalk_result(const DasNodeWalk* pThis); + +/** After a failure, the deepest container the walk searched. + * + * Only its URI: a node the walk loaded was not kept. A session can load + * it again cheaply and say "no child X under Y, which has ...". An empty + * string names the start node, when that is a root; it is the one + * container the caller still holds. + * + * @param psRest If not NULL, receives the part of the path below that + * container, valid for the life of the walk. + * @returns The URI, or NULL when the state is not das_walk_failed or no + * container was ever searched, as when no seed URL answered. + * @memberof DasNodeWalk */ +DAS_API const char* DasNodeWalk_nearest(const DasNodeWalk* pThis, const char** psRest); + +/** @} */ + +/** Free a node a constructor returned, and every node loaded below it. + * + * A child node, one a walk attached, is ignored: it belongs to its + * parent and is freed with it. * @memberof DasNode */ -DAS_API void del_RootNode(DasNode* pNode); +DAS_API void del_DasNode(DasNode* pNode); #ifdef __cplusplus } #endif -#endif /* _das_catalog_h_ */ +#endif /* _das_node_h_ */ diff --git a/das3/node_priv.h b/das3/node_priv.h new file mode 100644 index 0000000..9333041 --- /dev/null +++ b/das3/node_priv.h @@ -0,0 +1,186 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Node internals shared by node.c and catalog.c. Not installed. + * + * DasNode is opaque to applications so that the layout below can change + * after the v3.0 tag without breaking them. C has no package-private, so + * the two files that need the fields include this header and everything + * else goes through the accessors in node.h. Functions declared here have + * no DAS_API: the library is built with hidden visibility, so they are + * reachable inside libdas3 and nowhere else. + */ + +#ifndef _das_node_priv_h_ +#define _das_node_priv_h_ + +#include "node.h" + +/* How a node tree works, for readers of node.c and catalog.c. + * + * Derived types. struct das_node is the common header of every node. + * Nodes that have children (Catalog, Collection, SourceSet and the SPASE + * and SPDF trees) are really a DasCatNode, defined in node.c, whose first + * member is a DasNode so that either can be handled through a DasNode*. + * The type is checked before casting, see DasNode_isCatalog(). Terminal + * nodes (stream sources, file aggregations) are a bare struct das_node. + * + * Loading. Every node begins as a JSON document that _dasnode_fromJson() + * turns into a struct of the right type. Where the bytes come from is + * not node.c's business: catalog.c reads files, the cache and HTTP, a + * browser build hands them to new_DasNode_json(). The parsed DasJdo is + * kept in pDom for the life of the node and the accessors read from it + * rather than copying fields out. A DasNodeWalk resolves a path below a + * container by matching it against the names in the container dictionary + * and asking its driver for the matching child's URL; a child that arrives + * is remembered in the parent's pSubNodes array once the walk succeeds, + * so a second walk of the same path needs no document. + * + * Ownership. Trees are owned from the top. A node with bOwner set was + * handed to a caller by a constructor: it owns everything loaded below it + * and only del_DasNode() frees it. Callers never free a child node, it + * belongs to its parent's pSubNodes array. An owner need not be the top + * of any catalog: a seed loaded from a file, or a node a program kept + * after dropping the tree it was found in, since _DasNode_delTree() skips + * owners. An application holding site:/uiowa/juno/wav thus owns a one + * node tree with no parent in memory. + * + * Transport. None. A node holds no credentials, agent string or cache + * settings; the session that loads nodes, see catalog.c, keeps those and + * drives a DasNodeWalk with them. + */ + +struct das_node { + das_node_type_e nType; + char sURL[512]; /* where this node was loaded from */ + char sPath[512]; /* the node's catalog path URI, may be empty */ + bool bOwner; /* handed out by a constructor: owns the tree below + it, freed only by del_DasNode() */ + void* pDom; /* the parsed document, a DasJdo, current layout */ + void* pShim; /* elements added on load that the parser did not + allocate, see _dasnode_upgrade(). May be NULL */ +}; + +/* Catalog nodes have extra arrays to cache nodes */ +#define _D2C_SEP_SZ 8 +typedef struct das_catalog { + DasNode base; + + DasAry* pSubNodes; + + DasAry* pSubPaths; + + /* The json object or xml element tag which contains the sub elements */ + void* pContainer; + + /* The path separator, defaults to '/' unless overridden by the object */ + char sPathSep[_D2C_SEP_SZ]; + + /* The name of the element which contains sub elements */ + char sContainer[64]; + +} DasCatNode; + + + +/* Build a node from JSON text, see new_DasNode_json() for the public + * form. sUrl is where the text came from and sFinalUrl the same after + * redirects, both for messages. sPathUri may be NULL. The node is not + * marked as owned, the caller decides that. */ +DasNode* _dasnode_fromJson( + const char* pBytes, size_t uBytes, const char* sUrl, const char* sFinalUrl, + const char* sPathUri +); + +/* True if some child listed by pBase could be the next step of sSubPath, + * which starts with the separator pBase uses. Nothing is loaded. */ +bool _DasNode_subMatch(DasNode* pBase, const char* sSubPathWithSep); + +/* Frees a node and everything below it, except nodes a constructor handed + * to a caller: those are freed only by del_DasNode(). A child lives in + * its parent's pSubNodes cache, freeing one alone would leave a dangling + * entry there. */ +void _DasNode_delTree(DasNode* pThis); + +/* Catalog URI spellings. + * + * Names are composed top down: the das catalog root node has no separator, so + * its child is "tag:das2.org,2012:site", and that node's separator + * ":/" makes "tag:das2.org,2012:site:/uiowa". A root has no URI, so its + * children are named by their bare names and the levels below by the + * path from the root. The shorthands "site:" and "test:" follow the + * composition rule. Alone, one names the node: + * "site:" is "tag:das2.org,2012:site". With a path after it, it stands + * for the node and its separator, and a '/' is supplied or collapsed so + * that "site:uiowa" and "site:/uiowa" name one node and stamp one + * requisition. Nothing else is normalized: a full URI is copied as + * written, since separators inside a path belong to the catalog nodes + * and may not be '/'. Returns sOut, empty for a NULL input. */ +const char* _das_uri_expand(const char* sIn, char* sOut, size_t uOut); + +/* The mount table: a synthetic Catalog node whose children are other + * nodes placed at chosen catalog URIs. This is the machinery behind + * DasCat_mount() and DasCat_setSeed(). + * + * It works the way the das catalog index does. That node has + * "separator":null, so its children are named by whole URI prefixes such + * as "tag:das2.org,2012:" and a path cannot be split on a separator to + * find one. Instead the walk goes through the children in document order + * and takes the FIRST child whose name is a prefix of the requested path, + * then resolves the rest of the path below that child. + * + * A mount table is a small JSON document of that shape, written in memory + * and parsed like any downloaded node. Each child is named by the full + * URI of a mount and its urls list holds the file or URL to load there. + * For the two mounts (./uiowa.json at site:/uiowa) and (./wav.json at + * site:/uiowa/juno/wav) the document is in spirit: + * + * { "type":"Catalog", "separator":null, "catalog":{ + * "tag:das2.org,2012:site:/uiowa/juno/wav": {"urls":["./wav.json"]}, + * "tag:das2.org,2012:site:/uiowa": {"urls":["./uiowa.json"]} + * }} + * + * Resolving site:/uiowa/juno/wav/survey matches the wav child, loads + * wav.json and looks up "/survey" inside it. site:/uiowa/cassini skips + * wav and matches uiowa. Ordering matters because the first match wins, + * not the best match. If uiowa were listed before wav it would also + * claim every wav path and wav.json would never be consulted. Hence the + * children are written longest URI first, so a more specific mount + * shadows a broader one. Whether a URI no mount covers goes on to the + * federated catalog is the session's decision, made before the table is + * consulted at all. + * + * @param aUrl The URL or file path to load for each mount, in the same order + * as aUri. None may be NULL or empty. + * + * @param aUri The full catalog URI each mount sits at, for example + * "tag:das2.org,2012:site:/uiowa". None may be NULL or empty, and + * the "site:" shorthand is NOT expanded here, callers such as DasCat + * do that first. Order does not matter, the children are sorted. + * + * @param uMounts The number of entries in aUrl and aUri, at least 1 and at + * most 64, the size of the sort buffer in node.c. + * + * @returns A new mount table, owned by the caller, whose source URL reports + * as "memory:mounts", to be freed with del_DasNode(), or NULL if the + * arguments are bad + * or the document could not be allocated. Nothing is fetched at + * this point; the mounted nodes load on first lookup. + */ +DasNode* _dasnode_newMountTable(const char** aUrl, const char** aUri, size_t uMounts); + +#endif /* _das_node_priv_h_ */ diff --git a/das3/req.c b/das3/req.c new file mode 100644 index 0000000..3cb3306 --- /dev/null +++ b/das3/req.c @@ -0,0 +1,527 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* A requisition is strings all the way down, see req.h for why. Nothing + * in this file knows what a key means; that is source.c's job. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include + +#include "util.h" +#include "log.h" +#include "req_priv.h" + +/* ************************************************************************* */ +/* Helpers */ + +static bool _dasreq_strieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + while(*a && *b){ + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + ++a; ++b; + } + return (*a == '\0') && (*b == '\0'); +} + +/* das_strdup() returns NULL for NULL and for "", and here "" is a value: + an empty stamp, an unfilled entry. Every string in a requisition goes + through this so accessors can hand them straight back. */ +static char* _dasreq_dupz(const char* s) +{ + if((s == NULL)||(s[0] == '\0')) return (char*)calloc(1, 1); + return das_strdup(s); +} + +static bool _dasreq_setStr(char** ps, const char* s) +{ + char* sNew = _dasreq_dupz(s); + if(sNew == NULL) return false; + free(*ps); + *ps = sNew; + return true; +} + +bool _das_json_scalar(const DasJdo* p, char* sBuf, size_t uLen) +{ + if((p == NULL)||(uLen == 0)) return false; + switch(p->type){ + case das_json_type_str:{ + const char* s = ((const das_json_str*)p->value)->string; + strncpy(sBuf, s, uLen - 1); + sBuf[uLen - 1] = '\0'; + return true; + } + case das_json_type_num:{ + const das_json_num* pNum = (const das_json_num*)p->value; + size_t u = (pNum->number_size < uLen - 1) ? pNum->number_size : uLen - 1; + memcpy(sBuf, pNum->number, u); + sBuf[u] = '\0'; + return true; + } + case das_json_type_true: strncpy(sBuf, "true", uLen); sBuf[uLen-1] = '\0'; return true; + case das_json_type_false: strncpy(sBuf, "false", uLen); sBuf[uLen-1] = '\0'; return true; + default: return false; + } +} + +/* ************************************************************************* */ +/* Life cycle */ + +DasReq* new_DasReq(void) +{ + DasReq* pThis = (DasReq*)calloc(1, sizeof(DasReq)); + if(pThis == NULL) return NULL; + pThis->sPath = _dasreq_dupz(NULL); + pThis->sUrl = _dasreq_dupz(NULL); + pThis->sVersion = _dasreq_dupz(NULL); + pThis->sLabel = _dasreq_dupz(NULL); + if(!pThis->sPath || !pThis->sUrl || !pThis->sVersion || !pThis->sLabel){ + del_DasReq(pThis); + return NULL; + } + return pThis; +} + +void del_DasReq(DasReq* pThis) +{ + if(pThis == NULL) return; + DasReq_clear(pThis); + free(pThis->aEntry); + free(pThis->aRaw); + free(pThis->sPath); + free(pThis->sUrl); + free(pThis->sVersion); + free(pThis->sLabel); + free(pThis); +} + +DasReq* copy_DasReq(const DasReq* pThis) +{ + DasReq* pNew = new_DasReq(); + if(pNew == NULL) return NULL; + DasReq_setSrc(pNew, pThis->sPath, pThis->sUrl, pThis->sVersion); + DasReq_setLabel(pNew, pThis->sLabel); + for(size_t u = 0; u < pThis->uEntry; ++u){ + const struct das_req_entry* pE = pThis->aEntry + u; + int n = _DasReq_add(pNew, pE->sKey, pE->sVal, pE->bDerived); + if(n < 0){ del_DasReq(pNew); return NULL; } + _DasReq_setVerdict(pNew, (size_t)n, pE->verdict, pE->sWhy); + } + for(size_t u = 0; u < pThis->uRaw; ++u){ + if(DasReq_setRaw(pNew, pThis->aRaw[u].sName, pThis->aRaw[u].sVal) != DAS_OKAY){ + del_DasReq(pNew); return NULL; + } + } + return pNew; +} + +/* ************************************************************************* */ +/* Stamp and label */ + +void DasReq_setSrc(DasReq* pThis, const char* sPath, const char* sUrl, const char* sVersion) +{ + _dasreq_setStr(&pThis->sPath, sPath); + _dasreq_setStr(&pThis->sUrl, sUrl); + _dasreq_setStr(&pThis->sVersion, sVersion); +} + +const char* DasReq_srcPath(const DasReq* pThis){ return pThis->sPath; } +const char* DasReq_srcUrl(const DasReq* pThis){ return pThis->sUrl; } +const char* DasReq_srcVersion(const DasReq* pThis){ return pThis->sVersion; } +const char* DasReq_label(const DasReq* pThis){ return pThis->sLabel; } +void DasReq_setLabel(DasReq* pThis, const char* sLabel){ _dasreq_setStr(&pThis->sLabel, sLabel); } + +/* ************************************************************************* */ +/* Entries */ + +static void _dasreq_freeEntry(struct das_req_entry* pE) +{ + free(pE->sKey); + free(pE->sVal); + free(pE->sWhy); + memset(pE, 0, sizeof(struct das_req_entry)); +} + +int _DasReq_add(DasReq* pThis, const char* sKey, const char* sVal, bool bDerived) +{ + if(pThis->uEntry == pThis->uEntryCap){ + size_t uCap = pThis->uEntryCap ? pThis->uEntryCap * 2 : 8; + struct das_req_entry* a = (struct das_req_entry*)realloc( + pThis->aEntry, uCap * sizeof(struct das_req_entry) + ); + if(a == NULL) return -1; + pThis->aEntry = a; + pThis->uEntryCap = uCap; + } + struct das_req_entry* pE = pThis->aEntry + pThis->uEntry; + memset(pE, 0, sizeof(struct das_req_entry)); + pE->sKey = _dasreq_dupz(sKey); + pE->sVal = _dasreq_dupz(sVal); + if((pE->sKey == NULL)||(pE->sVal == NULL)){ _dasreq_freeEntry(pE); return -1; } + pE->bDerived = bDerived; + pE->nProp = -1; + pThis->uEntry += 1; + return (int)(pThis->uEntry - 1); +} + +void _DasReq_dropDerived(DasReq* pThis) +{ + size_t uKeep = 0; + for(size_t u = 0; u < pThis->uEntry; ++u){ + if(pThis->aEntry[u].bDerived){ _dasreq_freeEntry(pThis->aEntry + u); continue; } + if(uKeep != u) pThis->aEntry[uKeep] = pThis->aEntry[u]; + ++uKeep; + } + pThis->uEntry = uKeep; +} + +void _DasReq_setVerdict(DasReq* pThis, size_t uIdx, das_req_verdict_e verdict, const char* sWhy) +{ + if(uIdx >= pThis->uEntry) return; + struct das_req_entry* pE = pThis->aEntry + uIdx; + pE->verdict = verdict; + free(pE->sWhy); + pE->sWhy = (verdict == das_req_bad) ? _dasreq_dupz(sWhy) : NULL; +} + +void _DasReq_clearVerdicts(DasReq* pThis) +{ + for(size_t u = 0; u < pThis->uEntry; ++u){ + pThis->aEntry[u].verdict = das_req_unchecked; + pThis->aEntry[u].nProp = -1; + free(pThis->aEntry[u].sWhy); + pThis->aEntry[u].sWhy = NULL; + } +} + +int DasReq_find(const DasReq* pThis, const char* sKey) +{ + for(size_t u = 0; u < pThis->uEntry; ++u) + if(!pThis->aEntry[u].bDerived && _dasreq_strieq(pThis->aEntry[u].sKey, sKey)) return (int)u; + return -1; +} + +DasErrCode DasReq_set(DasReq* pThis, const char* sKey, const char* sVal) +{ + if((sKey == NULL)||(sKey[0] == '\0')){ + daslog_error("Requisition keys may not be empty"); + return DASERR_SRC; + } + /* Any change makes the last check stale for the derived entries, they + are the check's output */ + _DasReq_dropDerived(pThis); + + int n = DasReq_find(pThis, sKey); + if(n >= 0){ + struct das_req_entry* pE = pThis->aEntry + n; + char* sNew = _dasreq_dupz(sVal); + if(sNew == NULL) return DASERR_SRC; + free(pE->sVal); + pE->sVal = sNew; + pE->verdict = das_req_unchecked; + free(pE->sWhy); + pE->sWhy = NULL; + return DAS_OKAY; + } + return (_DasReq_add(pThis, sKey, sVal, false) < 0) ? DASERR_SRC : DAS_OKAY; +} + +const char* DasReq_value(const DasReq* pThis, const char* sKey) +{ + int n = DasReq_find(pThis, sKey); + return (n < 0) ? NULL : pThis->aEntry[n].sVal; +} + +bool DasReq_remove(DasReq* pThis, const char* sKey) +{ + int n = DasReq_find(pThis, sKey); + if(n < 0) return false; + _dasreq_freeEntry(pThis->aEntry + n); + memmove(pThis->aEntry + n, pThis->aEntry + n + 1, + (pThis->uEntry - n - 1) * sizeof(struct das_req_entry)); + pThis->uEntry -= 1; + _DasReq_dropDerived(pThis); + return true; +} + +void DasReq_clear(DasReq* pThis) +{ + for(size_t u = 0; u < pThis->uEntry; ++u) _dasreq_freeEntry(pThis->aEntry + u); + pThis->uEntry = 0; + for(size_t u = 0; u < pThis->uRaw; ++u){ + free(pThis->aRaw[u].sName); + free(pThis->aRaw[u].sVal); + } + pThis->uRaw = 0; +} + +size_t DasReq_count(const DasReq* pThis){ return pThis->uEntry; } + +const char* DasReq_key(const DasReq* pThis, size_t u) +{ + return (u < pThis->uEntry) ? pThis->aEntry[u].sKey : NULL; +} + +const char* DasReq_val(const DasReq* pThis, size_t u) +{ + return (u < pThis->uEntry) ? pThis->aEntry[u].sVal : NULL; +} + +das_req_verdict_e DasReq_verdict(const DasReq* pThis, size_t u) +{ + return (u < pThis->uEntry) ? pThis->aEntry[u].verdict : das_req_unchecked; +} + +const char* DasReq_why(const DasReq* pThis, size_t u) +{ + if((u >= pThis->uEntry)||(pThis->aEntry[u].sWhy == NULL)) return ""; + return pThis->aEntry[u].sWhy; +} + +bool DasReq_derived(const DasReq* pThis, size_t u) +{ + return (u < pThis->uEntry) ? pThis->aEntry[u].bDerived : false; +} + +/* ************************************************************************* */ +/* Raw parameters. Names are protocol names and compare exactly. */ + +DasErrCode DasReq_setRaw(DasReq* pThis, const char* sName, const char* sVal) +{ + if((sName == NULL)||(sName[0] == '\0')){ + daslog_error("Protocol parameter names may not be empty"); + return DASERR_SRC; + } + for(size_t u = 0; u < pThis->uRaw; ++u){ + if(strcmp(pThis->aRaw[u].sName, sName) == 0) + return _dasreq_setStr(&pThis->aRaw[u].sVal, sVal) ? DAS_OKAY : DASERR_SRC; + } + if(pThis->uRaw == pThis->uRawCap){ + size_t uCap = pThis->uRawCap ? pThis->uRawCap * 2 : 4; + struct das_req_raw* a = (struct das_req_raw*)realloc( + pThis->aRaw, uCap * sizeof(struct das_req_raw) + ); + if(a == NULL) return DASERR_SRC; + pThis->aRaw = a; + pThis->uRawCap = uCap; + } + struct das_req_raw* pR = pThis->aRaw + pThis->uRaw; + pR->sName = _dasreq_dupz(sName); + pR->sVal = _dasreq_dupz(sVal); + if((pR->sName == NULL)||(pR->sVal == NULL)){ + free(pR->sName); free(pR->sVal); + return DASERR_SRC; + } + pThis->uRaw += 1; + return DAS_OKAY; +} + +size_t DasReq_rawCount(const DasReq* pThis){ return pThis->uRaw; } + +const char* DasReq_raw(const DasReq* pThis, size_t u, const char** psVal) +{ + if(u >= pThis->uRaw) return NULL; + if(psVal) *psVal = pThis->aRaw[u].sVal; + return pThis->aRaw[u].sName; +} + +/* ************************************************************************* */ +/* Saving. A hand rolled writer because the document is three dicts of + * strings and das_json has no builder. */ + +typedef struct { char* s; size_t u; size_t uCap; bool bFail; } _out_t; + +static void _dasreq_put(_out_t* pO, const char* s) +{ + size_t n = strlen(s); + if(pO->u + n + 1 > pO->uCap){ + size_t uCap = (pO->uCap ? pO->uCap * 2 : 256); + while(uCap < pO->u + n + 1) uCap *= 2; + char* sNew = (char*)realloc(pO->s, uCap); + if(sNew == NULL){ pO->bFail = true; return; } + pO->s = sNew; + pO->uCap = uCap; + } + memcpy(pO->s + pO->u, s, n + 1); + pO->u += n; +} + +static void _dasreq_putStr(_out_t* pO, const char* s) +{ + _dasreq_put(pO, "\""); + char sEsc[8]; + for(const unsigned char* p = (const unsigned char*)s; *p; ++p){ + switch(*p){ + case '"': _dasreq_put(pO, "\\\""); break; + case '\\': _dasreq_put(pO, "\\\\"); break; + case '\n': _dasreq_put(pO, "\\n"); break; + case '\r': _dasreq_put(pO, "\\r"); break; + case '\t': _dasreq_put(pO, "\\t"); break; + default: + if(*p < 0x20){ snprintf(sEsc, sizeof(sEsc), "\\u%04x", *p); _dasreq_put(pO, sEsc); } + else{ sEsc[0] = (char)*p; sEsc[1] = '\0'; _dasreq_put(pO, sEsc); } + } + } + _dasreq_put(pO, "\""); +} + +static void _dasreq_putPair(_out_t* pO, const char* sKey, const char* sVal, bool bFirst) +{ + if(!bFirst) _dasreq_put(pO, ",\n"); + _dasreq_put(pO, " "); + _dasreq_putStr(pO, sKey); + _dasreq_put(pO, ": "); + _dasreq_putStr(pO, sVal); +} + +char* DasReq_text(const DasReq* pThis) +{ + _out_t o = {NULL, 0, 0, false}; + _dasreq_put(&o, "{\n \"source\": {\n"); + _dasreq_putPair(&o, "path", pThis->sPath, true); + _dasreq_putPair(&o, "url", pThis->sUrl, false); + _dasreq_putPair(&o, "version", pThis->sVersion, false); + _dasreq_put(&o, "\n },\n \"label\": "); + _dasreq_putStr(&o, pThis->sLabel); + _dasreq_put(&o, ",\n \"settings\": {\n"); + bool bFirst = true; + for(size_t u = 0; u < pThis->uEntry; ++u){ + if(pThis->aEntry[u].bDerived) continue; + _dasreq_putPair(&o, pThis->aEntry[u].sKey, pThis->aEntry[u].sVal, bFirst); + bFirst = false; + } + _dasreq_put(&o, "\n },\n \"params\": {\n"); + for(size_t u = 0; u < pThis->uRaw; ++u) + _dasreq_putPair(&o, pThis->aRaw[u].sName, pThis->aRaw[u].sVal, u == 0); + _dasreq_put(&o, "\n }\n}\n"); + if(o.bFail){ free(o.s); return NULL; } + return o.s; +} + +DasErrCode DasReq_write(const DasReq* pThis, const char* sPath) +{ + char* sText = DasReq_text(pThis); + if(sText == NULL){ + daslog_error("Couldn't allocate the requisition text"); + return DASERR_SRC; + } + FILE* pOut = fopen(sPath, "wb"); + if(pOut == NULL){ + daslog_error_v("Couldn't open %s for writing", sPath); + free(sText); + return DASERR_SRC; + } + bool bOk = (fputs(sText, pOut) >= 0); + fclose(pOut); + free(sText); + if(!bOk){ + daslog_error_v("Couldn't write %s", sPath); + return DASERR_SRC; + } + return DAS_OKAY; +} + +/* Example settings in nodes are written as JSON paths, + "coords/time/props/min". Saved requisitions may use either that or + the dotted key, so a slash path is folded to a key on the way in: + drop "props", join with dots. Section aliases are left for the + source's matcher, which accepts them anyway. */ +static void _dasreq_pathToKey(const char* sPath, char* sKey, size_t uLen) +{ + size_t u = 0; + const char* pBeg = sPath; + sKey[0] = '\0'; + while(*pBeg && (u + 2 < uLen)){ + const char* pEnd = strchr(pBeg, '/'); + size_t n = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); + if((n != 5)||(strncmp(pBeg, "props", 5) != 0)){ + if(n > uLen - u - 2) n = uLen - u - 2; + if(u > 0) sKey[u++] = '.'; + memcpy(sKey + u, pBeg, n); + u += n; + sKey[u] = '\0'; + } + if(pEnd == NULL) break; + pBeg = pEnd + 1; + } +} + +static const char* _dasreq_jstr(const DasJdo* p, const char* sPath) +{ + const DasJdo* pSub = p ? DasJdo_get(p, sPath) : NULL; + return ((pSub != NULL)&&(pSub->type == das_json_type_str)) ? DasJdo_string(pSub) : NULL; +} + +DasReq* new_DasReq_text(const char* sJson, size_t uLen) +{ + DasJdo* pDoc = das_json_parse(sJson, uLen); + if((pDoc == NULL)||(pDoc->type != das_json_type_dict)){ + daslog_error("Requisition text is not a JSON object"); + free(pDoc); + return NULL; + } + DasReq* pThis = new_DasReq(); + if(pThis == NULL){ free(pDoc); return NULL; } + + const DasJdo* pSrc = DasJdo_get(pDoc, "source"); + DasReq_setSrc(pThis, _dasreq_jstr(pSrc, "path"), _dasreq_jstr(pSrc, "url"), _dasreq_jstr(pSrc, "version")); + DasReq_setLabel(pThis, _dasreq_jstr(pDoc, "label")); + + char sKey[256], sVal[1024]; + bool bOk = true; + const DasJdo* pSet = DasJdo_get(pDoc, "settings"); + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSet); bOk && pEl; pEl = pEl->next){ + if(strchr(pEl->name->string, '/')) _dasreq_pathToKey(pEl->name->string, sKey, sizeof(sKey)); + else{ strncpy(sKey, pEl->name->string, sizeof(sKey) - 1); sKey[sizeof(sKey)-1] = '\0'; } + if(!_das_json_scalar(pEl->value, sVal, sizeof(sVal))) sVal[0] = '\0'; + bOk = (DasReq_set(pThis, sKey, sVal) == DAS_OKAY); + } + const DasJdo* pRaw = DasJdo_get(pDoc, "params"); + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pRaw); bOk && pEl; pEl = pEl->next){ + if(!_das_json_scalar(pEl->value, sVal, sizeof(sVal))) sVal[0] = '\0'; + bOk = (DasReq_setRaw(pThis, pEl->name->string, sVal) == DAS_OKAY); + } + free(pDoc); + if(!bOk){ del_DasReq(pThis); return NULL; } + return pThis; +} + +DasReq* new_DasReq_file(const char* sPath) +{ + FILE* pIn = fopen(sPath, "rb"); + if(pIn == NULL){ + daslog_error_v("Couldn't open requisition file %s", sPath); + return NULL; + } + fseek(pIn, 0, SEEK_END); + long nLen = ftell(pIn); + fseek(pIn, 0, SEEK_SET); + if(nLen < 0){ fclose(pIn); daslog_error_v("Couldn't size %s", sPath); return NULL; } + char* sBuf = (char*)malloc((size_t)nLen + 1); + if(sBuf == NULL){ fclose(pIn); return NULL; } + size_t uRead = fread(sBuf, 1, (size_t)nLen, pIn); + fclose(pIn); + sBuf[uRead] = '\0'; + DasReq* pThis = new_DasReq_text(sBuf, uRead); + free(sBuf); + return pThis; +} diff --git a/das3/req.h b/das3/req.h new file mode 100644 index 0000000..04e3549 --- /dev/null +++ b/das3/req.h @@ -0,0 +1,268 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/** @file req.h Requisitions: what a client wants from a data source */ + +#ifndef _das_req_h_ +#define _das_req_h_ + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup catalog + * @{ + */ + +/** @defgroup req Requisitions + * + * A requisition is the list of settings a client wants from a data + * source. It is the form an application fills in, and it is deliberately + * simple. A DasReq holds strings naming entries in the interface section + * of a data source node. Everything that interprets a requisition is a + * function of the DasSrc or it's derived classes. + * @see DasSrc_check() and DasStreamSrc_url() + * + * A requisition is self contained and may outlive the interface it was + * built from. It can be saved to a file one day, reloaded the next, and + * used to query the data source again. + * + * @par Entries and verdicts + * + * Each entry is a key, a value and a verdict. Keys are the dotted names + * described under @ref src_keys. Shorter spellings are not interpreted + * by the requisition; they are kept as entered and the DasSrc resolves + * them. Verdicts start out unchecked. DasSrc_check() validates every + * entry against a source, marking each one ok or bad and attaching a + * reason to the bad ones. + * + * Sources create requisitions. DasSrc_newReq() hands back one seeded + * with a worked example, or a blank one listing every required key with + * an empty value so that a form knows what must be filled. Either way + * the requisition is stamped with the source URI, and a later + * DasSrc_check() refuses to validate it against a different source. A + * requisition made with new_DasReq() has no stamp and is validated by + * whatever source it is handed to. + * + * @par Raw parameters + * + * A requisition may also carry raw protocol parameters: name and value + * pairs sent to the server exactly as given. These bypass the interface + * entirely and are for people who already know the server. The source + * applies them after everything else, so they win. This is not the + * intended use of a requisition, only an escape hatch. + * + * @par Saved requisitions + * + * DasReq_write() and new_DasReq_file() round trip a requisition through a + * small JSON document, the same shape as an example in a source node with + * a `source` block in front: + * + * @code + * { "source": { "path": "tag:das2.org,2012:site:/uiowa/juno/wav/survey", + * "url": "https://jupiter.physics.uiowa.edu/.../flex.json", + * "version": "0.7" }, + * "label": "Perijove 16, 30 second bins", + * "settings": { "coords/time/props/min": "2018-10-02T11:00", + * "coords/time/props/max": "2018-10-04T11:00", + * "coords/time/props/res": "30" }, + * "params": { "extra": "1" } } + * @endcode + * + * The path is the source's identity and is what a later check compares; + * the URL records where that copy of the node came from. The version is + * the node format version. + * + * @todo The catalog spec has no version for a source's interface, only + * for the node format. Until it does, a stale requisition may contain + * unknown keys at check time. + */ + +/** How an entry fared the last time a source looked at it */ +typedef enum das_req_verdict_enum { + das_req_unchecked = 0, /**< not validated since it was set */ + das_req_ok, /**< the key resolved and the value is well formed */ + das_req_bad /**< see DasReq_why() */ +} das_req_verdict_e; + +/** A requisition. Create with new_DasReq() or DasSrc_newReq(), free with + * del_DasReq(). Not thread safe for writers. */ +typedef struct das_req DasReq; + +/** An empty requisition with no source stamp */ +DAS_API DasReq* new_DasReq(void); + +/** A requisition read from the JSON text written by DasReq_text(). + * @returns a new requisition, or NULL after logging what was wrong with + * the text. */ +DAS_API DasReq* new_DasReq_text(const char* sJson, size_t uLen); + +/** A requisition read from a file written by DasReq_write(), or NULL + * after logging why not. */ +DAS_API DasReq* new_DasReq_file(const char* sPath); + +/** A deep copy, verdicts included */ +DAS_API DasReq* copy_DasReq(const DasReq* pThis); + +DAS_API void del_DasReq(DasReq* pThis); + +/* ************************************************************************* */ +/* The stamp and the label */ + +/** Record which source this requisition is for. + * + * DasSrc_newReq() does this for you. Call it yourself when adopting a + * requisition built from loose strings so that later checks hold it to + * one source. + * + * @param sPath The source's catalog URI, see DasSrc_path(). "" removes + * the stamp. + * @param sUrl Where the source document was loaded from, kept for + * diagnostics; it plays no part in matching. + * @param sVersion The node format version, or "". + * @memberof DasReq + */ +DAS_API void DasReq_setSrc( + DasReq* pThis, const char* sPath, const char* sUrl, const char* sVersion +); + +/** The catalog URI of the stamped source, "" when unstamped. + * @memberof DasReq */ +DAS_API const char* DasReq_srcPath(const DasReq* pThis); + +/** The URL of the stamped source, "" when unstamped. @memberof DasReq */ +DAS_API const char* DasReq_srcUrl(const DasReq* pThis); + +/** The node format version of the stamped source, "" when unknown. + * @memberof DasReq */ +DAS_API const char* DasReq_srcVersion(const DasReq* pThis); + +/** A short description, the example label when seeded from one, else + * "". @memberof DasReq */ +DAS_API const char* DasReq_label(const DasReq* pThis); + +/** @memberof DasReq */ +DAS_API void DasReq_setLabel(DasReq* pThis, const char* sLabel); + +/* ************************************************************************* */ +/* Entries */ + +/** Set a value. + * + * The key and value are stored as given. Nothing is resolved or checked + * here, there is no source to ask; the entry's verdict becomes unchecked + * and the next DasSrc_check() validates it. Setting a key that is already + * present replaces its value. Keys compare case insensitively. + * + * @param sKey Any spelling from @ref src_keys, or a coordinate name + * alone for the range forms described there. + * @param sVal The value. "" is allowed and means the key is wanted but + * not yet filled in, which is how a blank requisition lists what + * a source requires. NULL is treated as "". + * @returns DAS_OKAY, or DASERR_SRC if memory ran out or the key is empty. + * @memberof DasReq + */ +DAS_API DasErrCode DasReq_set(DasReq* pThis, const char* sKey, const char* sVal); + +/** The value stored under a key, exactly as set, or NULL when the key is + * not present. Only keys set directly are found, the derived entries a + * check adds are reached by index. @memberof DasReq */ +DAS_API const char* DasReq_value(const DasReq* pThis, const char* sKey); + +/** Drop an entry. @returns true if there was one. @memberof DasReq */ +DAS_API bool DasReq_remove(DasReq* pThis, const char* sKey); + +/** Drop every entry and raw parameter, keeping the stamp and label. + * @memberof DasReq */ +DAS_API void DasReq_clear(DasReq* pThis); + +/** The number of entries, including any a check derived, see + * DasReq_derived(). @memberof DasReq */ +DAS_API size_t DasReq_count(const DasReq* pThis); + +/** Entry number uIdx's key, or NULL past the end. @memberof DasReq */ +DAS_API const char* DasReq_key(const DasReq* pThis, size_t uIdx); + +/** Entry number uIdx's value, or NULL past the end. @memberof DasReq */ +DAS_API const char* DasReq_val(const DasReq* pThis, size_t uIdx); + +/** @memberof DasReq */ +DAS_API das_req_verdict_e DasReq_verdict(const DasReq* pThis, size_t uIdx); + +/** Why entry number uIdx was rejected, "" unless its verdict is + * das_req_bad. The text names the key and is meant to be shown as is. + * @memberof DasReq */ +DAS_API const char* DasReq_why(const DasReq* pThis, size_t uIdx); + +/** True for entries a check added rather than the client. + * + * A coordinate range given as one key, `time=2026-01-02,2026-01-03`, + * expands to entries for the minimum and maximum, and a required key the + * client never set is added with an empty value and a bad verdict so a + * form can mark it. Derived entries are rebuilt on every check and + * vanish on the next set; a panel usually skips them when drawing and a + * client reading values back should too. + * @memberof DasReq + */ +DAS_API bool DasReq_derived(const DasReq* pThis, size_t uIdx); + +/** Index of the entry for a key, or -1. @memberof DasReq */ +DAS_API int DasReq_find(const DasReq* pThis, const char* sKey); + +/* ************************************************************************* */ +/* Raw parameters */ + +/** Name a protocol parameter directly, see the group description. + * @returns DAS_OKAY, or DASERR_SRC if memory ran out. @memberof DasReq */ +DAS_API DasErrCode DasReq_setRaw(DasReq* pThis, const char* sName, const char* sVal); + +/** @memberof DasReq */ +DAS_API size_t DasReq_rawCount(const DasReq* pThis); + +/** Raw parameter number uIdx. + * @param psVal Receives the value, may be NULL. + * @returns The name, or NULL past the end. @memberof DasReq */ +DAS_API const char* DasReq_raw(const DasReq* pThis, size_t uIdx, const char** psVal); + +/* ************************************************************************* */ +/* Saving */ + +/** The requisition as JSON text, see the group description. + * + * Derived entries and verdicts are not written; they belong to a check, + * not to the requisition. + * + * @returns A heap string to be released with free(), or NULL if memory + * ran out. + * @memberof DasReq + */ +DAS_API char* DasReq_text(const DasReq* pThis); + +/** Write DasReq_text() to a file, replacing any existing file. + * @returns DAS_OKAY, or DASERR_SRC after logging why not. @memberof DasReq */ +DAS_API DasErrCode DasReq_write(const DasReq* pThis, const char* sPath); + +/** @} */ +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* _das_req_h_ */ diff --git a/das3/req_priv.h b/das3/req_priv.h new file mode 100644 index 0000000..d5ce652 --- /dev/null +++ b/das3/req_priv.h @@ -0,0 +1,87 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Requisition internals shared by req.c and source.c. Not installed. + * + * source.c is the only writer of verdicts and derived entries, so those + * operations are here rather than in req.h: an application has no + * business validating a requisition. + */ + +#ifndef _das_req_priv_h_ +#define _das_req_priv_h_ + +#include "req.h" +#include "json.h" + +struct das_req_entry { + char* sKey; /* owned, as the client typed it */ + char* sVal; /* owned, never NULL, may be "" */ + das_req_verdict_e verdict; + char* sWhy; /* owned, NULL unless bad */ + bool bDerived; + int nProp; /* index of the source property the last check + resolved this entry to, -1 if none. Only + meaningful to the source that set it. */ +}; + +struct das_req_raw { + char* sName; + char* sVal; +}; + +struct das_req { + char* sPath; /* stamp, each "" when unknown */ + char* sUrl; + char* sVersion; + char* sLabel; + + struct das_req_entry* aEntry; + size_t uEntry; + size_t uEntryCap; + + struct das_req_raw* aRaw; + size_t uRaw; + size_t uRawCap; +}; + +/* Append an entry without looking for a duplicate. Used for derived + * entries, which are keyed by full key and rebuilt wholesale, so the + * uniqueness DasReq_set() maintains for client keys is not needed. + * Returns the new index, or -1 if memory ran out. */ +int _DasReq_add(DasReq* pThis, const char* sKey, const char* sVal, bool bDerived); + +/* Drop every derived entry. Called at the start of a check so that a + * second check never sees its own output. */ +void _DasReq_dropDerived(DasReq* pThis); + +/* Set the verdict of an entry, with a reason for a bad one (ignored + * otherwise). sWhy is copied. */ +void _DasReq_setVerdict(DasReq* pThis, size_t uIdx, das_req_verdict_e verdict, const char* sWhy); + +/* Mark every entry unchecked. DasReq_set() does this for the one entry + * it touches; a check that is about to validate everything again calls this. */ +void _DasReq_clearVerdicts(DasReq* pThis); + +/* Copy a JSON scalar as text: strings as they are, numbers in their + * source notation, true and false as those words. Returns false for + * null, containers and missing values, leaving sBuf untouched. The one + * reader of JSON number text in the catalog code, since das_json keeps + * numbers as unterminated text and a default of 86.4 should stay 86.4. */ +bool _das_json_scalar(const DasJdo* pJdo, char* sBuf, size_t uLen); + +#endif /* _das_req_priv_h_ */ diff --git a/das3/source.c b/das3/source.c new file mode 100644 index 0000000..c0cb21c --- /dev/null +++ b/das3/source.c @@ -0,0 +1,1700 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Data sources, see source.h. + * + * Failures over content, a broken node or a bad user value, are logged + * and returned, never raised through das_error(), because this code + * meets bad catalogs and typos every day and the default error + * disposition would exit the program over either. Clients that catch + * log messages see everything; clients that do not still get verdicts. + * + * Grew out of the pilot in utilities/das_fetch.c, which carried the + * translation rules for two node layouts. Those rules are not written + * down anywhere else yet; until the catalog spec states them, the + * comments on _DasSrcCtrl_translate() are the reference. + */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include +#include + +#include "util.h" +#include "log.h" +#include "time.h" +#include "node.h" +#include "node_priv.h" +#include "src_priv.h" + +/* Section names as the current catalogs spell them. Keys are built from + * these; input accepts the singular forms too, see _das_src_secByName(). */ +static const char* g_sSecName[das_src_sec_size] = {"coords", "data", "formats", "options"}; + +/* Scalars copied out of the document are short in practice, the longest + * seen is a few dozen bytes; anything past this is cut rather than + * carried. */ +#define SCALAR_SZ 1024 + +/* ************************************************************************* */ +/* Small string and JSON helpers. DasJdo_get() takes slash paths, so the + * dots in parameter names such as "read.time.min" are not special to it. */ + +static bool _dassrc_strieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + while(*a && *b){ + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + ++a; ++b; + } + return (*a == '\0') && (*b == '\0'); +} + +static void _dassrc_strcpyz(char* sDest, const char* sSrc, size_t uDest) +{ + if(uDest == 0) return; + strncpy(sDest, sSrc ? sSrc : "", uDest - 1); + sDest[uDest - 1] = '\0'; +} + +static const DasJdo* _dassrc_jget(const DasJdo* p, const char* sPath) +{ + return ((p == NULL)||(sPath == NULL)) ? NULL : DasJdo_get(p, sPath); +} + +static const char* _dassrc_jstr(const DasJdo* p, const char* sPath) +{ + const DasJdo* pSub = _dassrc_jget(p, sPath); + return ((pSub != NULL)&&(pSub->type == das_json_type_str)) ? DasJdo_string(pSub) : NULL; +} + +static bool _dassrc_jisdict(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_dict); } +static bool _dassrc_jisary(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_ary); } +static bool _dassrc_jtrue(const DasJdo* p){ return (p != NULL)&&(p->type == das_json_type_true); } +static bool _dassrc_jisbool(const DasJdo* p){ + return (p != NULL)&&((p->type == das_json_type_true)||(p->type == das_json_type_false)); +} +static size_t _dassrc_jlen(const DasJdo* p) +{ + if(_dassrc_jisdict(p)) return ((const das_json_dict*)p->value)->length; + if(_dassrc_jisary(p)) return ((const das_json_ary*)p->value)->length; + return 0; +} + +/* das_strdup() returns NULL for "", which here would turn an empty default + into no default. Empty strings are copied as empty strings. */ +static char* _dassrc_strdupz(const char* s) +{ + if((s == NULL)||(s[0] == '\0')) return (char*)calloc(1, 1); + return das_strdup(s); +} + +/* Every string a source keeps, keys, default values, enum items and range + * ends, is a separate small allocation, freed with the source. A possible + * optimization, not taken: the source owns pDoc for its whole life, so + * plain JSON strings could be referenced in place and only numbers, which + * need formatting, copied. That would remove most of these allocations + * at the cost of a copied-or-borrowed flag per string. A full string + * arena was considered and rejected: it needs an exact size pass that + * duplicates the formatting logic below, and it hides overruns from + * valgrind. Revisit if a heap profile of a browser build points here. */ +static char* _dassrc_scalarDup(const DasJdo* p) +{ + char sBuf[SCALAR_SZ]; + if(!_das_json_scalar(p, sBuf, sizeof(sBuf))) return NULL; + return _dassrc_strdupz(sBuf); +} + +static bool _dassrc_parseBool(const char* s, bool* pOut) +{ + if(s == NULL) return false; + if(_dassrc_strieq(s,"true")||_dassrc_strieq(s,"yes")||_dassrc_strieq(s,"on")||_dassrc_strieq(s,"1")|| + _dassrc_strieq(s,"t")||_dassrc_strieq(s,"y")){ *pOut = true; return true; } + if(_dassrc_strieq(s,"false")||_dassrc_strieq(s,"no")||_dassrc_strieq(s,"off")||_dassrc_strieq(s,"0")|| + _dassrc_strieq(s,"f")||_dassrc_strieq(s,"n")){ *pOut = false; return true; } + return false; +} + +/* snprintf that appends and never runs the cursor past the buffer */ +static size_t _dassrc_catf(char* sBuf, size_t uLen, size_t u, const char* sFmt, ...) +{ + if(u >= uLen) return u; + va_list ap; + va_start(ap, sFmt); + int n = vsnprintf(sBuf + u, uLen - u, sFmt, ap); + va_end(ap); + if(n < 0) return u; + return (u + (size_t)n < uLen) ? u + (size_t)n : uLen; +} + +/* ************************************************************************* */ +/* Key grammar */ + +const char* das_src_sec_name(das_src_sec_e sec) +{ + return ((sec >= 0)&&(sec < das_src_sec_size)) ? g_sSecName[sec] : ""; +} + +bool _das_src_secByName(const char* s, das_src_sec_e* pSec) +{ + if(_dassrc_strieq(s,"coord")||_dassrc_strieq(s,"coords")||_dassrc_strieq(s,"coordinate")||_dassrc_strieq(s,"coordinates")){ + *pSec = das_src_sec_coord; return true; + } + if(_dassrc_strieq(s,"data")){ *pSec = das_src_sec_data; return true; } + if(_dassrc_strieq(s,"format")||_dassrc_strieq(s,"formats")||_dassrc_strieq(s,"fmt")){ + *pSec = das_src_sec_format; return true; + } + if(_dassrc_strieq(s,"option")||_dassrc_strieq(s,"options")||_dassrc_strieq(s,"opt")||_dassrc_strieq(s,"opts")){ + *pSec = das_src_sec_option; return true; + } + return false; +} + +const char* _das_src_propName(const char* s) +{ + if(s == NULL) return ""; + if(_dassrc_strieq(s, "minimum")||_dassrc_strieq(s, "min")) return "min"; + if(_dassrc_strieq(s, "maximum")||_dassrc_strieq(s, "max")) return "max"; + if(_dassrc_strieq(s, "resolution")||_dassrc_strieq(s, "res")) return "res"; + if(_dassrc_strieq(s, "interval")||_dassrc_strieq(s, "inter")) return "inter"; + if(_dassrc_strieq(s, "enable")||_dassrc_strieq(s, "enabled")) return "enabled"; + return s; +} + +bool _das_src_propEq(const char* a, const char* b) +{ + return _dassrc_strieq(_das_src_propName(a), _das_src_propName(b)); +} + +int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]) +{ + if(sKey == NULL) return 0; + int n = 0; + const char* pBeg = sKey; + while(true){ + const char* pEnd = strchr(pBeg, '.'); + size_t uLen = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); + if((uLen == 0)||(uLen >= DAS_MAX_ID_BUFSZ)||(n == 3)) return 0; + memcpy(aParts[n], pBeg, uLen); + aParts[n][uLen] = '\0'; + ++n; + if(pEnd == NULL) break; + pBeg = pEnd + 1; + } + return n; +} + +/* ************************************************************************* */ +/* Parsing the interface */ + +/* Fold a type word from a node to the enum. NULL and unknown words are + * text, the caller decides whether that is final. */ +static bool _dassrc_typeByName(const char* s, das_srcprop_type_e* pType) +{ + if(s == NULL) return false; + if(_dassrc_strieq(s,"boolean")||_dassrc_strieq(s,"bool")){ *pType = das_srcprop_bool; return true; } + if(_dassrc_strieq(s,"integer")||_dassrc_strieq(s,"int")){ *pType = das_srcprop_int; return true; } + if(_dassrc_strieq(s,"real")||_dassrc_strieq(s,"double")||_dassrc_strieq(s,"float")){ + *pType = das_srcprop_real; return true; + } + if(_dassrc_strieq(s,"isotime")||_dassrc_strieq(s,"time")||_dassrc_strieq(s,"datetime")){ + *pType = das_srcprop_time; return true; + } + if(_dassrc_strieq(s,"enum")){ *pType = das_srcprop_enum; return true; } + if(_dassrc_strieq(s,"string")||_dassrc_strieq(s,"text")){ *pType = das_srcprop_text; return true; } + return false; +} + +/* Preference order: a set enum list, the property's type, a boolean + * default, a flag toggle, the protocol parameter's type, the shape of the + * value, then text. The enum list goes first because a property declared + * "string" with a list of choices is still a list of choices to anyone + * presenting it. A flag set parameter type is a server side packing + * detail and says nothing about the single toggle that feeds it. */ +static void _DasSrcProp_resolveType(DasSrcProp* p) +{ + p->sTypeName = _dassrc_jstr(p->pJdo, "type"); + if(p->sTypeName == NULL) p->sTypeName = ""; + const DasSrcCtrl* pC = p->pCtrl; + const DasJdo* pVal = _dassrc_jget(p->pJdo, "value"); + + if(pC && (pC->uEnum > 0)){ p->type = das_srcprop_enum; return; } + if(_dassrc_typeByName(p->sTypeName, &(p->type))){ + if(p->type != das_srcprop_enum) return; + } + if(_dassrc_jisbool(pVal) || (pC && (_dassrc_jstr(pC->pSet, "flag") != NULL))){ + p->type = das_srcprop_bool; return; + } + const char* sParamType = pC ? _dassrc_jstr(pC->pParam, "type") : NULL; + if(_dassrc_typeByName(sParamType, &(p->type)) && (p->type != das_srcprop_enum)){ + if(p->sTypeName[0] == '\0') p->sTypeName = sParamType; + return; + } + if((pVal != NULL)&&(pVal->type == das_json_type_num)){ p->type = das_srcprop_real; return; } + p->type = das_srcprop_text; +} + +/* Enum entries are usually dicts carrying value plus per item pval or flag + * overrides, but a bare scalar list is accepted too. */ +static bool _DasSrcCtrl_fillEnum(DasSrcCtrl* p) +{ + const DasJdo* pEnum = _dassrc_jget(p->pSet, "enum"); + if(!_dassrc_jisary(pEnum)) return true; + + size_t uMax = _dassrc_jlen(pEnum); + if(uMax == 0) return true; + p->aEnum = (char**)calloc(uMax, sizeof(char*)); + if(p->aEnum == NULL) return false; + + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ + const DasJdo* pVal = _dassrc_jisdict(pEl->value) ? _dassrc_jget(pEl->value, "value") : pEl->value; + char* s = _dassrc_scalarDup(pVal); + if(s == NULL) continue; + p->aEnum[p->uEnum] = s; + p->uEnum += 1; + } + return true; +} + +static void _DasSrcCtrl_fillRange(DasSrcCtrl* p) +{ + const DasSrcProp* pProp = p->pProp; + const DasJdo* pRange = _dassrc_jget(p->pSet, "range"); + if(!_dassrc_jisary(pRange)) pRange = _dassrc_jget(p->pParam, "range"); + if(!_dassrc_jisary(pRange) && (pProp->sec == das_src_sec_coord) && + (_das_src_propEq(pProp->sName, "min") || _das_src_propEq(pProp->sName, "max"))) + pRange = _dassrc_jget(pProp->pGroup->pJdo, "validRange"); + if(!_dassrc_jisary(pRange)) return; + + const das_json_ary_el* pEl = DasJdo_aryFirst(pRange); + if(pEl == NULL) return; + p->sLo = _dassrc_scalarDup(pEl->value); + if(pEl->next) p->sHi = _dassrc_scalarDup(pEl->next->value); +} + +/* Release what a control owns; the control itself lives in aCtrl */ +static void _DasSrcCtrl_clear(DasSrcCtrl* p) +{ + if(p == NULL) return; + for(size_t v = 0; v < p->uEnum; ++v) free(p->aEnum[v]); + free(p->aEnum); + free(p->sLo); + free(p->sHi); +} + +/* A member of a group dict is a property if it is a dict that looks like + * one. In the current layout every member of "props" qualifies; in the + * 2018 layout properties sit beside the group's name and title + * strings, and a units dict with a value counts as the units property + * just as it does under props today. */ +static bool _dassrc_isPropDict(const DasJdo* p) +{ + if(!_dassrc_jisdict(p)) return false; + return (_dassrc_jget(p, "value") != NULL)||(_dassrc_jget(p, "set") != NULL)||(_dassrc_jget(p, "type") != NULL); +} + +static const DasJdo* _dassrc_groupProps(const DasJdo* pGroup) +{ + const DasJdo* pProps = _dassrc_jget(pGroup, "props"); + return _dassrc_jisdict(pProps) ? pProps : pGroup; +} + +/* Count and fill share one walk. With pSrc NULL only the counts are + * wanted. */ +typedef struct { + size_t uGroup; size_t uProp; size_t uCtrl; + DasSrc* pSrc; const DasJdo* pParams; bool bFail; +} _walk_t; + +/* A property is settable when its set block names a protocol parameter. + * The counting pass and the fill pass must agree on this, so it is one + * function. */ +static const DasJdo* _dassrc_setBlock(const DasJdo* pJdo) +{ + const DasJdo* pSet = _dassrc_jget(pJdo, "set"); + return (_dassrc_jisdict(pSet) && (_dassrc_jstr(pSet, "param") != NULL)) ? pSet : NULL; +} + +static void _dassrc_walkProp(_walk_t* pW, DasSrcGroup* pG, const char* sName, const DasJdo* pJdo) +{ + if(pW->pSrc == NULL){ + pW->uProp += 1; + if(_dassrc_setBlock(pJdo) != NULL) pW->uCtrl += 1; + return; + } + if(pW->bFail) return; + + DasSrc* pS = pW->pSrc; + DasSrcProp* p = pS->aProp + pW->uProp; + memset(p, 0, sizeof(DasSrcProp)); + p->sec = pG->sec; + p->pGroup = pG; + p->sName = sName; + p->pJdo = pJdo; + + size_t uKey = strlen(g_sSecName[pG->sec]) + strlen(pG->sName) + strlen(sName) + 3; + p->sKey = (char*)malloc(uKey); + if(p->sKey == NULL){ pW->bFail = true; return; } + if(pG->sName[0] == '\0') + snprintf(p->sKey, uKey, "%s.%s", g_sSecName[pG->sec], sName); + else + snprintf(p->sKey, uKey, "%s.%s.%s", g_sSecName[pG->sec], pG->sName, sName); + + p->sValue = _dassrc_scalarDup(_dassrc_jget(pJdo, "value")); + + const DasJdo* pSet = _dassrc_jget(pJdo, "set"); + if(_dassrc_jisdict(pSet)){ + const char* sParam = _dassrc_jstr(pSet, "param"); + if(sParam == NULL){ + daslog_warn_v("Property %s has a set block that names no parameter, " + "treating it as fixed", p->sKey); + } + else{ + /* Controls live in one block sized by the counting pass */ + if(pW->uCtrl >= pS->uCtrl){ pW->bFail = true; return; } + DasSrcCtrl* pC = pS->aCtrl + pW->uCtrl; + pW->uCtrl += 1; + memset(pC, 0, sizeof(DasSrcCtrl)); + pC->pProp = p; + pC->pSet = pSet; + pC->sParam = sParam; + pC->pParam = _dassrc_jget(pW->pParams, sParam); + pC->bRequired = _dassrc_jtrue(_dassrc_jget(pSet, "required")) || _dassrc_jtrue(_dassrc_jget(pC->pParam, "required")); + p->pCtrl = pC; + if(!_DasSrcCtrl_fillEnum(pC)){ pW->bFail = true; return; } + _DasSrcCtrl_fillRange(pC); + } + } + _DasSrcProp_resolveType(p); + + if(_das_src_propEq(sName, "enabled") && (pG->nEnable < 0)){ + pG->nEnable = (int)(pW->uProp - pG->uFirst); + pG->sXor = _dassrc_jstr(pJdo, "xorGroup"); + } + pW->uProp += 1; + pG->uCount += 1; +} + +static void _dassrc_walkGroup(_walk_t* pW, das_src_sec_e sec, const char* sName, const DasJdo* pJdo) +{ + if(!_dassrc_jisdict(pJdo)) return; + DasSrcGroup* pG = NULL; + if(pW->pSrc != NULL){ + pG = pW->pSrc->aGroup + pW->uGroup; + memset(pG, 0, sizeof(DasSrcGroup)); + pG->pSrc = pW->pSrc; + pG->sec = sec; + pG->sName = sName; + pG->pJdo = pJdo; + pG->uFirst = pW->uProp; + pG->nEnable = -1; + + const DasJdo* pMime = _dassrc_jget(pJdo, "mimeTypes"); + if(_dassrc_jisary(pMime) && (_dassrc_jlen(pMime) > 0)){ + pG->aMime = (const char**)calloc(_dassrc_jlen(pMime), sizeof(char*)); + if(pG->aMime == NULL){ pW->bFail = true; return; } + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pMime); pEl; pEl = pEl->next){ + if(DasJdo_string(pEl->value) == NULL) continue; + pG->aMime[pG->uMime] = DasJdo_string(pEl->value); + pG->uMime += 1; + } + } + pW->pSrc->aSecCount[sec] += 1; + } + pW->uGroup += 1; + + const DasJdo* pProps = _dassrc_groupProps(pJdo); + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pProps); pEl; pEl = pEl->next){ + if(!_dassrc_isPropDict(pEl->value)) continue; + _dassrc_walkProp(pW, pG, pEl->name->string, pEl->value); + } +} + +/* Groups of a section in presentation order: "time" first for coordinates, + * otherwise as the node lists them. Forms want time on top and nodes do + * not always put it there. */ +static void _dassrc_walkSection(_walk_t* pW, das_src_sec_e sec, const DasJdo* pSec) +{ + if(!_dassrc_jisdict(pSec)) return; + if(pW->pSrc != NULL) pW->pSrc->aSecFirst[sec] = pW->uGroup; + if(sec == das_src_sec_coord){ + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next) + if(_dassrc_strieq(pEl->name->string, "time")) _dassrc_walkGroup(pW, sec, pEl->name->string, pEl->value); + } + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pSec); pEl; pEl = pEl->next){ + if((sec == das_src_sec_coord) && _dassrc_strieq(pEl->name->string, "time")) continue; + _dassrc_walkGroup(pW, sec, pEl->name->string, pEl->value); + } +} + +static void _dassrc_walkIface(_walk_t* pW, const DasJdo* pIface) +{ + const DasJdo* pCoords = _dassrc_jget(pIface, "coords"); + if(pCoords == NULL) pCoords = _dassrc_jget(pIface, "coordinates"); + _dassrc_walkSection(pW, das_src_sec_coord, pCoords); + _dassrc_walkSection(pW, das_src_sec_data, _dassrc_jget(pIface, "data")); + _dassrc_walkSection(pW, das_src_sec_format, _dassrc_jget(pIface, "formats")); + + /* Options are one group with no enclosing section */ + const DasJdo* pOpts = _dassrc_jget(pIface, "options"); + if(_dassrc_jisdict(pOpts)){ + if(pW->pSrc != NULL) pW->pSrc->aSecFirst[das_src_sec_option] = pW->uGroup; + _dassrc_walkGroup(pW, das_src_sec_option, "", pOpts); + } +} + +/* ************************************************************************* */ +/* Examples, parsed into requisitions when the source is built */ + +/* "coords/time/props/min" -> "coords.time.min". The section token is + * folded to the current spelling so a saved requisition reads the same + * whatever node it came from. */ +static bool _dassrc_exPathToKey(const char* sPath, char* sKey, size_t uLen) +{ + size_t u = 0; + int nTok = 0; + const char* pBeg = sPath; + sKey[0] = '\0'; + while(*pBeg){ + const char* pEnd = strchr(pBeg, '/'); + size_t n = pEnd ? (size_t)(pEnd - pBeg) : strlen(pBeg); + char sTok[DAS_MAX_ID_BUFSZ]; + if((n == 0)||(n >= sizeof(sTok))) return false; + memcpy(sTok, pBeg, n); sTok[n] = '\0'; + const char* sOut = sTok; + das_src_sec_e sec; + if(strcmp(sTok, "props") == 0) sOut = NULL; + else if((nTok == 0) && _das_src_secByName(sTok, &sec)) sOut = g_sSecName[sec]; + if(sOut != NULL){ + if(u + strlen(sOut) + 2 >= uLen) return false; + u = _dassrc_catf(sKey, uLen, u, "%s%s", nTok ? "." : "", sOut); + ++nTok; + } + if(pEnd == NULL) break; + pBeg = pEnd + 1; + } + return nTok > 0; +} + +/* The flag value a control sends, without any prefix: the parameter's + * flags entry for it, or the flag name itself */ +static const char* _DasSrcCtrl_flagVal(const DasSrcCtrl* pC, const char* sFlag, char* sBuf, size_t uLen) +{ + char sPath[DAS_MAX_ID_BUFSZ + 8]; + snprintf(sPath, sizeof(sPath), "flags/%s", sFlag); + const DasJdo* pF = _dassrc_jget(pC->pParam, sPath); + if(!_das_json_scalar(_dassrc_jget(pF, "value"), sBuf, uLen)) _dassrc_strcpyz(sBuf, sFlag, uLen); + return sBuf; +} + +/* A 2018 example is raw protocol parameters. Each is mapped back onto + * the control that would have produced it, so the requisition looks the + * same as one from a current node. Parameters no control explains are + * kept as raw parameters, nothing is dropped. */ +static bool _DasSrc_exampleFromParams(DasSrc* pThis, DasReq* pReq, const DasJdo* pParams) +{ + char sVal[SCALAR_SZ], sFlagVal[SCALAR_SZ], sPVal[SCALAR_SZ]; + for(const das_json_dict_el* pP = DasJdo_dictFirst(pParams); pP; pP = pP->next){ + const char* sName = pP->name->string; + if(!_das_json_scalar(pP->value, sVal, sizeof(sVal))) continue; + + /* the controls feeding this parameter */ + bool bFlagSet = false, bHandled = false; + for(size_t u = 0; u < pThis->uProp; ++u){ + const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; + if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; + if(_dassrc_jstr(pC->pSet, "flag") != NULL){ bFlagSet = true; break; } + } + + if(bFlagSet){ + const char* sSep = NULL; + for(size_t u = 0; (sSep == NULL) && (u < pThis->uProp); ++u){ + const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; + if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; + sSep = _dassrc_jstr(pC->pParam, "flagSep"); + if(sSep == NULL) sSep = _dassrc_jstr(pC->pParam, "flag_sep"); + } + if((sSep == NULL)||(sSep[0] == '\0')) sSep = " "; + + char sLeft[SCALAR_SZ] = {'\0'}; /* tokens no control owns */ + char* pTok = sVal; + while(pTok && *pTok){ + char* pNext = strstr(pTok, sSep); + if(pNext){ *pNext = '\0'; pNext += strlen(sSep); } + bool bFound = false; + for(size_t u = 0; !bFound && (u < pThis->uProp); ++u){ + const DasSrcCtrl* pC = pThis->aProp[u].pCtrl; + if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; + const char* sFlag = _dassrc_jstr(pC->pSet, "flag"); + if(sFlag == NULL) continue; + _DasSrcCtrl_flagVal(pC, sFlag, sFlagVal, sizeof(sFlagVal)); + if(_dassrc_strieq(sFlagVal, pTok)){ + if(DasReq_set(pReq, pThis->aProp[u].sKey, "true") != DAS_OKAY) return false; + bFound = true; + } + } + if(!bFound && (pTok[0] != '\0')){ + size_t n = strlen(sLeft); + snprintf(sLeft + n, sizeof(sLeft) - n, "%s%s", n ? sSep : "", pTok); + } + pTok = pNext; + } + if(sLeft[0] != '\0') + if(DasReq_setRaw(pReq, sName, sLeft) != DAS_OKAY) return false; + continue; + } + + for(size_t u = 0; !bHandled && (u < pThis->uProp); ++u){ + const DasSrcProp* p = pThis->aProp + u; + const DasSrcCtrl* pC = p->pCtrl; + if((pC == NULL)||(strcmp(pC->sParam, sName) != 0)) continue; + + const DasJdo* pEnum = _dassrc_jget(pC->pSet, "enum"); + if(_dassrc_jisary(pEnum)){ + /* which choice sends this protocol value? */ + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); !bHandled && pEl; pEl = pEl->next){ + const DasJdo* pItem = _dassrc_jisdict(pEl->value) ? _dassrc_jget(pEl->value, "value") : pEl->value; + char sItem[SCALAR_SZ]; + if(!_das_json_scalar(pItem, sItem, sizeof(sItem))) continue; + bool bHit = _das_json_scalar(_dassrc_jget(pEl->value, "pval"), sPVal, sizeof(sPVal)) ? + _dassrc_strieq(sPVal, sVal) : _dassrc_strieq(sItem, sVal); + if(bHit){ + if(DasReq_set(pReq, p->sKey, sItem) != DAS_OKAY) return false; + bHandled = true; + } + } + continue; + } + if(_das_json_scalar(_dassrc_jget(pC->pSet, "pval"), sPVal, sizeof(sPVal))){ + /* a fixed protocol value stands for one state of a toggle */ + if(_dassrc_strieq(sPVal, sVal)){ + bool bOn = true; + const DasJdo* pTrig = _dassrc_jget(pC->pSet, "value"); + if(_dassrc_jisbool(pTrig)) bOn = _dassrc_jtrue(pTrig); + if(DasReq_set(pReq, p->sKey, bOn ? "true" : "false") != DAS_OKAY) return false; + bHandled = true; + } + continue; + } + if(DasReq_set(pReq, p->sKey, sVal) != DAS_OKAY) return false; + bHandled = true; + } + if(!bHandled) + if(DasReq_setRaw(pReq, sName, sVal) != DAS_OKAY) return false; + } + return true; +} + +static bool _DasSrc_loadExamples(DasSrc* pThis) +{ + const DasJdo* pEx = _dassrc_jget(pThis->pIface, "examples"); + const DasJdo* pProto = _dassrc_jget(pThis->pDoc, "protocol"); + bool bLegacyEx = false; + if(!_dassrc_jisary(pEx)){ + pEx = _dassrc_jget(pProto, "examples"); + bLegacyEx = _dassrc_jisdict(pEx); + if(!bLegacyEx) return true; + } + size_t uMax = _dassrc_jlen(pEx); + if(uMax == 0) return true; + pThis->aExample = (DasReq**)calloc(uMax, sizeof(DasReq*)); + if(pThis->aExample == NULL) return false; + + char sKey[256], sVal[SCALAR_SZ]; + if(bLegacyEx){ + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pEx); pEl; pEl = pEl->next){ + DasReq* pReq = new_DasReq(); + if(pReq == NULL) return false; + pThis->aExample[pThis->uExample] = pReq; + pThis->uExample += 1; + const char* sLabel = _dassrc_jstr(pEl->value, "title"); + if(sLabel == NULL) sLabel = _dassrc_jstr(pEl->value, "name"); + if(sLabel == NULL) sLabel = pEl->name->string; + DasReq_setLabel(pReq, sLabel); + DasSrc_stamp(pThis, pReq); + const DasJdo* pParams = _dassrc_jget(pEl->value, "http_params"); + if(_dassrc_jisdict(pParams) && !_DasSrc_exampleFromParams(pThis, pReq, pParams)) return false; + } + return true; + } + + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEx); pEl; pEl = pEl->next){ + DasReq* pReq = new_DasReq(); + if(pReq == NULL) return false; + pThis->aExample[pThis->uExample] = pReq; + pThis->uExample += 1; + const char* sLabel = _dassrc_jstr(pEl->value, "label"); + if(sLabel == NULL) sLabel = _dassrc_jstr(pEl->value, "title"); + if(sLabel == NULL) sLabel = _dassrc_jstr(pEl->value, "name"); + DasReq_setLabel(pReq, sLabel); + DasSrc_stamp(pThis, pReq); + const DasJdo* pSettings = _dassrc_jget(pEl->value, "settings"); + for(const das_json_dict_el* pP = DasJdo_dictFirst(pSettings); pP; pP = pP->next){ + if(!_dassrc_exPathToKey(pP->name->string, sKey, sizeof(sKey))){ + daslog_warn_v("Ignoring example setting '%s' in %s, the path makes no sense", + pP->name->string, pThis->sUrl); + continue; + } + if(!_das_json_scalar(pP->value, sVal, sizeof(sVal))) continue; + if(DasReq_set(pReq, sKey, sVal) != DAS_OKAY) return false; + } + } + return true; +} + +/* ************************************************************************* */ +/* Life cycle */ + +void del_DasSrc(DasSrc* pThis) +{ + if(pThis == NULL) return; + for(size_t u = 0; u < pThis->uExample; ++u) del_DasReq(pThis->aExample[u]); + free(pThis->aExample); + for(size_t u = 0; u < pThis->uProp; ++u){ + free(pThis->aProp[u].sKey); + free(pThis->aProp[u].sValue); + _DasSrcCtrl_clear(pThis->aProp[u].pCtrl); + } + free(pThis->aProp); + free(pThis->aCtrl); + for(size_t u = 0; u < pThis->uGroup; ++u) free((void*)pThis->aGroup[u].aMime); /* quiet msvc */ + free(pThis->aGroup); + free(pThis->pDoc); + free(pThis->sPath); + free(pThis->sUrl); + free(pThis); +} + +DasSrc* new_DasSrc_json(const char* sDoc, size_t uLen, const char* sPathUri, const char* sUrl) +{ + const char* sWhere = sUrl ? sUrl : (sPathUri ? sPathUri : "(memory)"); + DasJdo* pDoc = das_json_parse(sDoc, uLen); + if(!_dassrc_jisdict(pDoc)){ + daslog_error_v("Source document %s is not a JSON object", sWhere); + free(pDoc); + return NULL; + } + const char* sType = _dassrc_jstr(pDoc, "type"); + if(!_dassrc_strieq(sType, "HttpStreamSrc")){ + daslog_error_v("Node %s is of type %s, only HttpStreamSrc nodes describe data sources " + "this library can query", sWhere, sType ? sType : "(none)"); + free(pDoc); + return NULL; + } + + DasStreamSrc* pStream = (DasStreamSrc*)calloc(1, sizeof(DasStreamSrc)); + if(pStream == NULL){ free(pDoc); return NULL; } + DasSrc* pThis = &(pStream->base); + pThis->kind = das_src_kind_stream; + pThis->pDoc = pDoc; + + /* A source given no URI has none, and a requisition it makes carries + * no stamp, see DasReq_setSrc(): nothing can hold it to one source */ + char sUri[600] = {'\0'}; + if((sPathUri != NULL) && (sPathUri[0] != '\0')) + _das_uri_expand(sPathUri, sUri, sizeof(sUri)); + pThis->sPath = _dassrc_strdupz(sUri); + pThis->sUrl = _dassrc_strdupz(sUrl); + if((pThis->sPath == NULL)||(pThis->sUrl == NULL)){ del_DasSrc(pThis); return NULL; } + + pThis->pIface = _dassrc_jget(pDoc, "interface"); + if(!_dassrc_jisdict(pThis->pIface)) pThis->pIface = NULL; + + pStream->pProto = _dassrc_jget(pDoc, "protocol"); + if(!_dassrc_jisdict(pStream->pProto)) pStream->pProto = NULL; + pStream->pParams = _dassrc_jget(pStream->pProto, "httpParams"); + if(pStream->pParams == NULL) pStream->pParams = _dassrc_jget(pStream->pProto, "http_params"); + pStream->pBaseUrls = _dassrc_jget(pStream->pProto, "baseUrls"); + if(pStream->pBaseUrls == NULL) pStream->pBaseUrls = _dassrc_jget(pStream->pProto, "base_urls"); + if(!_dassrc_jisary(pStream->pBaseUrls)) pStream->pBaseUrls = NULL; + pThis->bLegacy = (_dassrc_jget(pStream->pProto, "http_params") != NULL) || + (_dassrc_jget(pThis->pIface, "coordinates") != NULL); + + /* Two passes over the same walk: count, allocate exactly that many, + * fill. Groups, properties and controls are three blocks, so a source + * is a handful of allocations however large its interface, which + * matters on a heap that never shrinks, as in a browser build. */ + _walk_t w = {0, 0, 0, NULL, pStream->pParams, false}; + _dassrc_walkIface(&w, pThis->pIface); + pThis->aGroup = (DasSrcGroup*)calloc(w.uGroup ? w.uGroup : 1, sizeof(DasSrcGroup)); + pThis->aProp = (DasSrcProp*)calloc(w.uProp ? w.uProp : 1, sizeof(DasSrcProp)); + pThis->aCtrl = (DasSrcCtrl*)calloc(w.uCtrl ? w.uCtrl : 1, sizeof(DasSrcCtrl)); + if((pThis->aGroup == NULL)||(pThis->aProp == NULL)||(pThis->aCtrl == NULL)){ + del_DasSrc(pThis); + return NULL; + } + size_t uGroups = w.uGroup, uProps = w.uProp, uCtrls = w.uCtrl; + pThis->uCtrl = uCtrls; /* the fill pass checks against it */ + + w.uGroup = 0; w.uProp = 0; w.uCtrl = 0; w.pSrc = pThis; + _dassrc_walkIface(&w, pThis->pIface); + pThis->uGroup = w.uGroup; + pThis->uProp = w.uProp; + pThis->uCtrl = w.uCtrl; + if(w.bFail || (uGroups != w.uGroup) || (uProps != w.uProp) || (uCtrls != w.uCtrl)){ + daslog_error_v("Couldn't build the interface of source %s", sWhere); + del_DasSrc(pThis); + return NULL; + } + + if(!_DasSrc_loadExamples(pThis)){ + daslog_error_v("Couldn't read the examples of source %s", sWhere); + del_DasSrc(pThis); + return NULL; + } + return pThis; +} + +DasSrc* new_DasSrc_node(const DasNode* pNode) +{ + if(pNode == NULL) return NULL; + if(!DasNode_isStreamSrc(pNode)){ + /* Name the children so a caller can pick one; picking is not done + here because the library has no basis for the choice. */ + const char* aNames[8]; + size_t uKids = dassrc_loadable(pNode, aNames, 8); + char sMsg[512]; + size_t n = snprintf(sMsg, sizeof(sMsg), "Node %s is not a data source", DasNode_srcUrl(pNode)); + if(uKids > 0) n = _dassrc_catf(sMsg, sizeof(sMsg), n, ", its loadable children are:"); + for(size_t u = 0; (u < uKids) && (u < 8); ++u) n = _dassrc_catf(sMsg, sizeof(sMsg), n, " %s", aNames[u]); + daslog_error_v("%s", sMsg); + return NULL; + } + const DasJdo* pDom = DasNode_getJdo(pNode, NULL); + if(pDom == NULL) return NULL; + + /* A serialized copy keeps the source independent of the tree. Nodes + are a few KB, so this is cheaper than sharing with a reference count + would be to get right. */ + size_t uLen = 0; + char* sText = (char*)DasJdo_writeMinified(pDom, &uLen); + if(sText == NULL){ + daslog_error_v("Couldn't copy the document of node %s", DasNode_srcUrl(pNode)); + return NULL; + } + DasSrc* pThis = new_DasSrc_json(sText, uLen, DasNode_pathUri(pNode), DasNode_srcUrl(pNode)); + free(sText); + return pThis; +} + +/* ************************************************************************* */ +/* Identity */ + +das_src_kind_e DasSrc_kind(const DasSrc* pThis){ return pThis->kind; } +const char* DasSrc_path(const DasSrc* pThis){ return pThis->sPath; } +const char* DasSrc_url(const DasSrc* pThis){ return pThis->sUrl; } + +static const char* _DasSrc_docStr(const DasSrc* pThis, const char* sPath) +{ + const char* s = _dassrc_jstr(pThis->pDoc, sPath); + return s ? s : ""; +} + +const char* DasSrc_name(const DasSrc* pThis) +{ + const char* s = _dassrc_jstr(pThis->pDoc, "label"); + if(s == NULL) s = _dassrc_jstr(pThis->pDoc, "name"); + return s ? s : ""; +} + +const char* DasSrc_title(const DasSrc* pThis){ return _DasSrc_docStr(pThis, "title"); } +const char* DasSrc_version(const DasSrc* pThis){ return _DasSrc_docStr(pThis, "version"); } + +bool DasSrc_needsAuth(const DasSrc* pThis) +{ + const DasJdo* pProto = _dassrc_jget(pThis->pDoc, "protocol"); + const DasJdo* pAuth = _dassrc_jget(pProto, "authorization"); + if(pAuth == NULL) pAuth = _dassrc_jget(pProto, "authentication"); + return _dassrc_jtrue(_dassrc_jget(pAuth, "required")); +} + +static const DasJdo* _DasSrc_contacts(const DasSrc* pThis) +{ + const DasJdo* p = _dassrc_jget(pThis->pDoc, "contacts"); + if(!_dassrc_jisary(p)) p = _dassrc_jget(pThis->pDoc, "tech_contacts"); + return _dassrc_jisary(p) ? p : NULL; +} + +size_t DasSrc_contactCount(const DasSrc* pThis){ return _dassrc_jlen(_DasSrc_contacts(pThis)); } + +const char* DasSrc_contact(const DasSrc* pThis, size_t uIdx, const char** psEmail) +{ + const das_json_ary_el* pEl = DasJdo_aryFirst(_DasSrc_contacts(pThis)); + for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; + if(pEl == NULL) return NULL; + const char* sName = _dassrc_jstr(pEl->value, "name"); + const char* sEmail = _dassrc_jstr(pEl->value, "email"); + if(psEmail) *psEmail = sEmail ? sEmail : ""; + return sName ? sName : ""; +} + +const DasJdo* DasSrc_jdo(const DasSrc* pThis, const char* sFragment) +{ + if((sFragment == NULL)||(sFragment[0] == '\0')) return pThis->pDoc; + return DasJdo_get(pThis->pDoc, sFragment); +} + +/* ************************************************************************* */ +/* Groups */ + +size_t DasSrc_groupCount(const DasSrc* pThis, das_src_sec_e sec) +{ + return ((sec >= 0)&&(sec < das_src_sec_size)) ? pThis->aSecCount[sec] : 0; +} + +const DasSrcGroup* DasSrc_group(const DasSrc* pThis, das_src_sec_e sec, size_t uIdx) +{ + if(uIdx >= DasSrc_groupCount(pThis, sec)) return NULL; + return pThis->aGroup + pThis->aSecFirst[sec] + uIdx; +} + +const DasSrcGroup* DasSrc_findGroup(const DasSrc* pThis, das_src_sec_e sec, const char* sName) +{ + if(sName == NULL) sName = ""; + for(size_t u = 0; u < DasSrc_groupCount(pThis, sec); ++u){ + const DasSrcGroup* pG = DasSrc_group(pThis, sec, u); + if(_dassrc_strieq(pG->sName, sName)) return pG; + } + return NULL; +} + +das_src_sec_e DasSrcGroup_section(const DasSrcGroup* pThis){ return pThis->sec; } +const char* DasSrcGroup_name(const DasSrcGroup* pThis){ return pThis->sName; } + +const char* DasSrcGroup_label(const DasSrcGroup* pThis) +{ + const char* s = _dassrc_jstr(pThis->pJdo, "label"); + if(s == NULL) s = _dassrc_jstr(pThis->pJdo, "name"); + return s ? s : pThis->sName; +} + +const char* DasSrcGroup_title(const DasSrcGroup* pThis) +{ + const char* s = _dassrc_jstr(pThis->pJdo, "title"); + return s ? s : ""; +} + +const DasSrcProp* DasSrcGroup_prop(const DasSrcGroup* pThis, size_t uIdx) +{ + return (uIdx < pThis->uCount) ? pThis->pSrc->aProp + pThis->uFirst + uIdx : NULL; +} + +const DasSrcProp* DasSrcGroup_enable(const DasSrcGroup* pThis) +{ + return (pThis->nEnable < 0) ? NULL : DasSrcGroup_prop(pThis, (size_t)pThis->nEnable); +} + +const char* DasSrcGroup_xorGroup(const DasSrcGroup* pThis){ return pThis->sXor; } +size_t DasSrcGroup_mimeCount(const DasSrcGroup* pThis){ return pThis->uMime; } + +const char* DasSrcGroup_mime(const DasSrcGroup* pThis, size_t uIdx) +{ + return (uIdx < pThis->uMime) ? pThis->aMime[uIdx] : NULL; +} + +size_t DasSrcGroup_propCount(const DasSrcGroup* pThis){ return pThis->uCount; } + +const DasSrcProp* DasSrcGroup_findProp(const DasSrcGroup* pThis, const char* sName) +{ + for(size_t u = 0; u < pThis->uCount; ++u){ + const DasSrcProp* p = DasSrcGroup_prop(pThis, u); + if(_das_src_propEq(p->sName, sName)) return p; + } + return NULL; +} + +size_t DasSrcGroup_ctrlCount(const DasSrcGroup* pThis) +{ + size_t n = 0; + for(size_t u = 0; u < pThis->uCount; ++u) + if(DasSrcGroup_prop(pThis, u)->pCtrl != NULL) ++n; + return n; +} + +/* ************************************************************************* */ +/* Properties */ + +size_t DasSrc_propCount(const DasSrc* pThis){ return pThis->uProp; } + +const DasSrcProp* DasSrc_prop(const DasSrc* pThis, size_t uIdx) +{ + return (uIdx < pThis->uProp) ? pThis->aProp + uIdx : NULL; +} + +das_src_sec_e DasSrcProp_section(const DasSrcProp* pThis){ return pThis->sec; } +const DasSrcGroup* DasSrcProp_group(const DasSrcProp* pThis){ return pThis->pGroup; } +const char* DasSrcProp_name(const DasSrcProp* pThis){ return pThis->sName; } +const char* DasSrcProp_key(const DasSrcProp* pThis){ return pThis->sKey; } +das_srcprop_type_e DasSrcProp_type(const DasSrcProp* pThis){ return pThis->type; } +const char* DasSrcProp_typeName(const DasSrcProp* pThis){ return pThis->sTypeName; } +const char* DasSrcProp_value(const DasSrcProp* pThis){ return pThis->sValue; } +const DasSrcCtrl* DasSrcProp_ctrl(const DasSrcProp* pThis){ return pThis->pCtrl; } + +/* Units for display. Current nodes put a units string on the property; + * both layouts may carry a units property with a "value" at the group + * level. */ +const char* DasSrcProp_units(const DasSrcProp* pThis) +{ + const DasJdo* pU = _dassrc_jget(pThis->pJdo, "units"); + if(pU != NULL){ + if(pU->type == das_json_type_str) return DasJdo_string(pU); + if(_dassrc_jstr(pU, "value")) return _dassrc_jstr(pU, "value"); + } + if(pThis->sec == das_src_sec_coord){ + const char* s = _dassrc_jstr(_dassrc_groupProps(pThis->pGroup->pJdo), "units/value"); + if(s != NULL) return s; + } + if(pThis->type == das_srcprop_time) return "UTC"; + return ""; +} + +const char* DasSrcProp_title(const DasSrcProp* pThis, char* sBuf, size_t uLen) +{ + const char* s = _dassrc_jstr(pThis->pJdo, "title"); + if(s == NULL) s = _dassrc_jstr(pThis->pJdo, "summary"); + if(s == NULL) s = _dassrc_jstr(pThis->pJdo, "label"); + if(s == NULL) s = _dassrc_jstr(pThis->pJdo, "name"); + if(s != NULL) return s; + if(pThis->sec == das_src_sec_option) return ""; + + const char* sGroup = DasSrcGroup_label(pThis->pGroup); + const char* sProp = _das_src_propName(pThis->sName); + if(strcmp(sProp, "enabled") == 0){ + if(pThis->sec == das_src_sec_format){ + snprintf(sBuf, uLen, "Output as %s", sGroup); + } + else{ + /* A variable's title is usually the physical quantity, which + reads better after "Output" than its short label does */ + const char* sTitle = _dassrc_jstr(pThis->pGroup->pJdo, "title"); + snprintf(sBuf, uLen, "Output %s", sTitle ? sTitle : sGroup); + } + return sBuf; + } + if(pThis->sec == das_src_sec_coord){ + const char* sFmt = NULL; + if(strcmp(sProp, "min") == 0) sFmt = "Minimum %s value to output"; + else if(strcmp(sProp, "max") == 0) sFmt = "Maximum %s value to output"; + else if(strcmp(sProp, "res") == 0) sFmt = "Average over %s bins of this width"; + else if(strcmp(sProp, "inter") == 0) sFmt = "Step between output %s values"; + else if(strcmp(sProp, "units") == 0) sFmt = "Units of %s values"; + if(sFmt != NULL){ + snprintf(sBuf, uLen, sFmt, sGroup); + return sBuf; + } + } + snprintf(sBuf, uLen, "%s %s", sGroup, pThis->sName); + return sBuf; +} + +/* ************************************************************************* */ +/* Controls */ + +const DasSrcProp* DasSrcCtrl_prop(const DasSrcCtrl* pThis){ return pThis->pProp; } +bool DasSrcCtrl_required(const DasSrcCtrl* pThis){ return pThis->bRequired; } +const char* DasSrcCtrl_param(const DasSrcCtrl* pThis){ return pThis->sParam; } +size_t DasSrcCtrl_enumCount(const DasSrcCtrl* pThis){ return pThis->uEnum; } + +const char* DasSrcCtrl_enum(const DasSrcCtrl* pThis, size_t uIdx) +{ + return (uIdx < pThis->uEnum) ? pThis->aEnum[uIdx] : NULL; +} + +bool DasSrcCtrl_range(const DasSrcCtrl* pThis, const char** psLo, const char** psHi) +{ + if((pThis->sLo == NULL)&&(pThis->sHi == NULL)) return false; + if(psLo) *psLo = pThis->sLo ? pThis->sLo : ""; + if(psHi) *psHi = pThis->sHi ? pThis->sHi : ""; + return true; +} + +DasErrCode DasSrcCtrl_check(const DasSrcCtrl* pThis, const char* sVal, char* sWhy, size_t uWhy) +{ + if(sVal == NULL) sVal = ""; + if(sWhy && uWhy) sWhy[0] = '\0'; + char sDummy[1]; + if(sWhy == NULL){ sWhy = sDummy; uWhy = 1; } + const char* sKey = pThis->pProp->sKey; + char* pEnd = NULL; + + switch(pThis->pProp->type){ + case das_srcprop_bool:{ + bool b; + if(_dassrc_parseBool(sVal, &b)) return DAS_OKAY; + snprintf(sWhy, uWhy, "'%s' is not a boolean value for %s, use true or false", sVal, sKey); + return DASERR_SRC; + } + case das_srcprop_enum:{ + for(size_t u = 0; u < pThis->uEnum; ++u) + if(_dassrc_strieq(pThis->aEnum[u], sVal)) return DAS_OKAY; + size_t u = snprintf(sWhy, uWhy, "'%s' is not an allowed value for %s, choose one of:", sVal, sKey); + for(size_t v = 0; v < pThis->uEnum; ++v) + u = _dassrc_catf(sWhy, uWhy, u, "%s %s", v ? "," : "", pThis->aEnum[v]); + return DASERR_SRC; + } + case das_srcprop_int: + strtol(sVal, &pEnd, 10); + if((pEnd != sVal) && (*pEnd == '\0')) return DAS_OKAY; + snprintf(sWhy, uWhy, "'%s' is not an integer value for %s", sVal, sKey); + return DASERR_SRC; + case das_srcprop_real: + strtod(sVal, &pEnd); + if((pEnd != sVal) && (*pEnd == '\0')) return DAS_OKAY; + snprintf(sWhy, uWhy, "'%s' is not a real value for %s", sVal, sKey); + return DASERR_SRC; + case das_srcprop_time:{ + /* "now" is how nodes spell an open ended range and servers take + it in requests too */ + das_time dt; + if(_dassrc_strieq(sVal, "now") || dt_parsetime(&dt, sVal)) return DAS_OKAY; + snprintf(sWhy, uWhy, "'%s' is not a time value for %s, use ISO 8601 such as " + "2026-01-15T12:00", sVal, sKey); + return DASERR_SRC; + } + default: + return DAS_OKAY; + } +} + +/* ************************************************************************* */ +/* Key lookup */ + +int _DasSrc_match(const DasSrc* pThis, const char* sKey, const DasSrcProp** aOut, int nMax) +{ + char aParts[3][DAS_MAX_ID_BUFSZ]; + int nParts = _das_src_splitKey(sKey, aParts); + if(nParts == 0) return 0; + + int n = 0; + das_src_sec_e sec; + const DasSrcProp* p = NULL; + +#define _ADD_MATCH if(n < nMax) aOut[n] = p; ++n; + + if(nParts == 3){ + if(!_das_src_secByName(aParts[0], &sec)) return 0; + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if((p->sec == sec) && _dassrc_strieq(p->pGroup->sName, aParts[1]) && + _das_src_propEq(p->sName, aParts[2])){ _ADD_MATCH } + } + return n; + } + + if(nParts == 2){ + /* "option.X" is a section qualified option, "SEC.GROUP" toggles a + variable, anything else is GROUP.PROP in any section */ + if(_das_src_secByName(aParts[0], &sec)){ + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if(p->sec != sec) continue; + if((sec == das_src_sec_option) && _das_src_propEq(p->sName, aParts[1])){ _ADD_MATCH } + else if((sec != das_src_sec_option) && _dassrc_strieq(p->pGroup->sName, aParts[1]) && + _das_src_propEq(p->sName, "enabled")){ _ADD_MATCH } + } + if(n > 0) return n; + } + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if(_dassrc_strieq(p->pGroup->sName, aParts[0]) && _das_src_propEq(p->sName, aParts[1])){ _ADD_MATCH } + } + return n; + } + + /* One part: an option, or a variable/format toggle */ + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if((p->sec == das_src_sec_option) && _das_src_propEq(p->sName, aParts[0])){ _ADD_MATCH } + else if((p->sec != das_src_sec_option) && _dassrc_strieq(p->pGroup->sName, aParts[0]) && + _das_src_propEq(p->sName, "enabled")){ _ADD_MATCH } + } + if(n > 0) return n; + + /* Last resort, a bare property name. Unique in practice for things + like 'serial', ambiguous for 'fracSecs' shared by das and csv, and + the ambiguity message then lists the full keys. */ + for(size_t u = 0; u < pThis->uProp; ++u){ + p = pThis->aProp + u; + if((p->sec != das_src_sec_option) && _das_src_propEq(p->sName, aParts[0])){ _ADD_MATCH } + } +#undef _ADD_MATCH + return n; +} + +const DasSrcGroup* _DasSrc_coordGroup(const DasSrc* pThis, const char* sKey) +{ + char aParts[3][DAS_MAX_ID_BUFSZ]; + int nParts = _das_src_splitKey(sKey, aParts); + const char* sGroup = NULL; + das_src_sec_e sec; + if(nParts == 1) sGroup = aParts[0]; + else if((nParts == 2) && _das_src_secByName(aParts[0], &sec) && (sec == das_src_sec_coord)) + sGroup = aParts[1]; + else return NULL; + return DasSrc_findGroup(pThis, das_src_sec_coord, sGroup); +} + +#define MAX_MATCH 8 + +static void _dassrc_ambiguous(const char* sKey, const DasSrcProp** aMatch, int n, char* sWhy, size_t uWhy) +{ + size_t u = snprintf(sWhy, uWhy, "Key '%s' is ambiguous, use one of:", sKey); + for(int i = 0; (i < n) && (i < MAX_MATCH); ++i) + u = _dassrc_catf(sWhy, uWhy, u, " %s", aMatch[i]->sKey); +} + +const DasSrcProp* DasSrc_findProp(const DasSrc* pThis, const char* sKey, char* sWhy, size_t uWhy) +{ + char sDummy[1]; + if(sWhy == NULL){ sWhy = sDummy; uWhy = 1; } + sWhy[0] = '\0'; + const DasSrcProp* aMatch[MAX_MATCH]; + int n = _DasSrc_match(pThis, sKey, aMatch, MAX_MATCH); + if(n == 1) return aMatch[0]; + if(n == 0) snprintf(sWhy, uWhy, "No property named '%s' in this source", sKey); + else _dassrc_ambiguous(sKey, aMatch, n, sWhy, uWhy); + return NULL; +} + +/* ************************************************************************* */ +/* Requisitions */ + +size_t DasSrc_exampleCount(const DasSrc* pThis){ return pThis->uExample; } + +const char* DasSrc_exampleLabel(const DasSrc* pThis, size_t uIdx) +{ + return (uIdx < pThis->uExample) ? DasReq_label(pThis->aExample[uIdx]) : NULL; +} + +void DasSrc_stamp(const DasSrc* pThis, DasReq* pReq) +{ + DasReq_setSrc(pReq, pThis->sPath, pThis->sUrl, DasSrc_version(pThis)); +} + +DasReq* DasSrc_newReq(const DasSrc* pThis, int nExample) +{ + if(nExample >= 0){ + if((size_t)nExample >= pThis->uExample){ + daslog_error_v("Source %s has %zu example(s), there is no example %d", + pThis->sUrl, pThis->uExample, nExample); + return NULL; + } + return copy_DasReq(pThis->aExample[nExample]); + } + DasReq* pReq = new_DasReq(); + if(pReq == NULL) return NULL; + DasSrc_stamp(pThis, pReq); + for(size_t u = 0; u < pThis->uProp; ++u){ + const DasSrcProp* p = pThis->aProp + u; + if((p->pCtrl == NULL)||!p->pCtrl->bRequired||(p->sValue != NULL)) continue; + if(DasReq_set(pReq, p->sKey, "") != DAS_OKAY){ del_DasReq(pReq); return NULL; } + } + return pReq; +} + +/* Identity is the catalog URI alone. The URL is where one copy of the + * node happened to come from; a mirror or a local mount of the same node + * has a different one and must validate the same requisitions. */ +static bool _DasSrc_stampMatches(const DasSrc* pThis, const DasReq* pReq) +{ + const char* sPath = DasReq_srcPath(pReq); + if(sPath[0] == '\0') return true; /* unstamped */ + return _dassrc_strieq(sPath, pThis->sPath); +} + +/* One item of a coordinate range: find the control, check the value, + * add the derived entry. Returns false with sWhy filled on any trouble. */ +static bool _dassrc_rangeItem( + const DasSrc* pThis, DasReq* pReq, const DasSrcGroup* pG, const char* sProp, + const char* sVal, bool* aPresent, bool* aCovered, char* sWhy, size_t uWhy +){ + const DasSrcProp* p = DasSrcGroup_findProp(pG, sProp); + if((p == NULL)||(p->pCtrl == NULL)){ + snprintf(sWhy, uWhy, "Coordinate %s has no settable '%s' property", pG->sName, sProp); + return false; + } + if(DasSrcCtrl_check(p->pCtrl, sVal, sWhy, uWhy) != DAS_OKAY) return false; + int n = _DasReq_add(pReq, p->sKey, sVal, true); + if(n < 0){ snprintf(sWhy, uWhy, "Out of memory"); return false; } + size_t uProp = (size_t)(p - pThis->aProp); + pReq->aEntry[n].nProp = (int)uProp; + _DasReq_setVerdict(pReq, (size_t)n, das_req_ok, NULL); + aPresent[uProp] = true; + aCovered[uProp] = true; + return true; +} + +/* GROUP=MIN,MAX[,STEP] or GROUP=SPAN, see "Keys" in source.h */ +static bool _dassrc_rangeEntry( + const DasSrc* pThis, DasReq* pReq, const DasSrcGroup* pG, const char* sVal, + bool* aPresent, bool* aCovered, char* sWhy, size_t uWhy +){ + if(strchr(sVal, ',') == NULL){ + /* a single time denotes its span */ + const DasSrcProp* pMin = DasSrcGroup_findProp(pG, "min"); + if((pMin == NULL)||(pMin->type != das_srcprop_time)){ + snprintf(sWhy, uWhy, "Give coordinate %s a range as MIN,MAX", pG->sName); + return false; + } + das_time dtBeg, dtEnd; + if(!dt_parsespan(sVal, &dtBeg, &dtEnd)){ + snprintf(sWhy, uWhy, "'%s' is neither a MIN,MAX range nor a time span such " + "as 2026-01-02 or 2026-01-02T14", sVal); + return false; + } + char sBeg[40], sEnd[40]; + dt_isoc(&dtBeg, sBeg, sizeof(sBeg), 0); + dt_isoc(&dtEnd, sEnd, sizeof(sEnd), 0); + return _dassrc_rangeItem(pThis, pReq, pG, "min", sBeg, aPresent, aCovered, sWhy, uWhy) && + _dassrc_rangeItem(pThis, pReq, pG, "max", sEnd, aPresent, aCovered, sWhy, uWhy); + } + + const char* aProps[3] = {"min", "max", NULL}; + aProps[2] = DasSrcGroup_findProp(pG, "res") ? "res" : "inter"; + + char sCopy[SCALAR_SZ]; + _dassrc_strcpyz(sCopy, sVal, sizeof(sCopy)); + char* pTok = sCopy; + for(int i = 0; i < 4; ++i){ + char* pComma = strchr(pTok, ','); + if(pComma) *pComma = '\0'; + if(i == 3){ + snprintf(sWhy, uWhy, "Too many values for coordinate %s, at most MIN,MAX,STEP", pG->sName); + return false; + } + if(pTok[0] != '\0') + if(!_dassrc_rangeItem(pThis, pReq, pG, aProps[i], pTok, aPresent, aCovered, sWhy, uWhy)) + return false; + if(pComma == NULL) break; + pTok = pComma + 1; + } + return true; +} + +DasErrCode DasSrc_check(const DasSrc* pThis, DasReq* pReq, size_t* puBad) +{ + if(puBad) *puBad = 0; + if(!_DasSrc_stampMatches(pThis, pReq)){ + daslog_error_v("Requisition is for source %s%s, not %s%s", + DasReq_srcPath(pReq), DasReq_srcPath(pReq)[0] ? "" : DasReq_srcUrl(pReq), + pThis->sPath, pThis->sPath[0] ? "" : pThis->sUrl); + return DASERR_SRC; + } + _DasReq_dropDerived(pReq); + _DasReq_clearVerdicts(pReq); + + /* present: some entry names the property, even badly. covered: a good + non-empty value was given for it. */ + size_t uProp = pThis->uProp ? pThis->uProp : 1; + bool* aPresent = (bool*)calloc(uProp * 2, sizeof(bool)); + if(aPresent == NULL) return DASERR_SRC; + bool* aCovered = aPresent + uProp; + + size_t uBad = 0; + char sWhy[1024]; + size_t uClient = pReq->uEntry; /* derived entries are appended past here */ + for(size_t u = 0; u < uClient; ++u){ + const char* sKey = pReq->aEntry[u].sKey; + const char* sVal = pReq->aEntry[u].sVal; + + const DasSrcGroup* pG = _DasSrc_coordGroup(pThis, sKey); + if(pG != NULL){ + if(_dassrc_rangeEntry(pThis, pReq, pG, sVal, aPresent, aCovered, sWhy, sizeof(sWhy))) + _DasReq_setVerdict(pReq, u, das_req_ok, NULL); + else{ + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); + ++uBad; + } + continue; + } + + const DasSrcProp* aMatch[MAX_MATCH]; + int n = _DasSrc_match(pThis, sKey, aMatch, MAX_MATCH); + if(n == 0){ + snprintf(sWhy, sizeof(sWhy), "No property named '%s' in this source", sKey); + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; + } + if(n > 1){ + _dassrc_ambiguous(sKey, aMatch, n, sWhy, sizeof(sWhy)); + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; + } + const DasSrcProp* p = aMatch[0]; + size_t uP = (size_t)(p - pThis->aProp); + pReq->aEntry[u].nProp = (int)uP; + if(p->pCtrl == NULL){ + snprintf(sWhy, sizeof(sWhy), "%s is a fixed property of this source and cannot be set", p->sKey); + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; + } + aPresent[uP] = true; + if(sVal[0] == '\0'){ + if(p->pCtrl->bRequired && (p->sValue == NULL)){ + snprintf(sWhy, sizeof(sWhy), "%s is required, give it a value", p->sKey); + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; + } + else + _DasReq_setVerdict(pReq, u, das_req_ok, NULL); + continue; + } + if(DasSrcCtrl_check(p->pCtrl, sVal, sWhy, sizeof(sWhy)) != DAS_OKAY){ + _DasReq_setVerdict(pReq, u, das_req_bad, sWhy); ++uBad; continue; + } + aCovered[uP] = true; + _DasReq_setVerdict(pReq, u, das_req_ok, NULL); + } + + /* An empty entry for a required control is fine after all when a range + entry covered the same control: the blank requisition lists + coords.time.min, the user typed time=A,B, both name one property */ + for(size_t u = 0; u < uClient; ++u){ + struct das_req_entry* pE = pReq->aEntry + u; + if((pE->verdict != das_req_bad)||(pE->nProp < 0)||(pE->sVal[0] != '\0')) continue; + if(!aCovered[pE->nProp]) continue; + _DasReq_setVerdict(pReq, u, das_req_ok, NULL); + --uBad; + } + + /* required controls nobody mentioned */ + for(size_t u = 0; u < pThis->uProp; ++u){ + const DasSrcProp* p = pThis->aProp + u; + if((p->pCtrl == NULL)||!p->pCtrl->bRequired||(p->sValue != NULL)||aPresent[u]) continue; + int n = _DasReq_add(pReq, p->sKey, "", true); + if(n < 0) break; + pReq->aEntry[n].nProp = (int)u; + snprintf(sWhy, sizeof(sWhy), "%s is required, set it with %s=VALUE", p->sKey, p->sKey); + _DasReq_setVerdict(pReq, (size_t)n, das_req_bad, sWhy); + ++uBad; + } + free(aPresent); + if(puBad) *puBad = uBad; + return uBad ? DASERR_SRC : DAS_OKAY; +} + +/* The format group a requisition turns on, else the node's default */ +static const DasSrcGroup* _DasSrc_chosenFormat(const DasSrc* pThis, const DasReq* pReq) +{ + const DasSrcGroup* pDef = NULL; + for(size_t g = 0; g < DasSrc_groupCount(pThis, das_src_sec_format); ++g){ + const DasSrcGroup* pG = DasSrc_group(pThis, das_src_sec_format, g); + const DasSrcProp* pOn = DasSrcGroup_enable(pG); + if(pOn == NULL) continue; + bool b = false; + if((pDef == NULL) && _dassrc_parseBool(pOn->sValue, &b) && b) pDef = pG; + if(pReq == NULL) continue; + for(size_t u = 0; u < pReq->uEntry; ++u){ + if(pReq->aEntry[u].bDerived) continue; + const DasSrcProp* aMatch[1]; + if(_DasSrc_match(pThis, pReq->aEntry[u].sKey, aMatch, 1) != 1) continue; + if((aMatch[0] == pOn) && _dassrc_parseBool(pReq->aEntry[u].sVal, &b) && b) return pG; + } + } + return pDef; +} + +const char* DasSrc_mime(const DasSrc* pThis, const DasReq* pReq) +{ + const DasSrcGroup* pG = _DasSrc_chosenFormat(pThis, pReq); + if((pG != NULL) && (pG->uMime > 0)) return pG->aMime[0]; + + /* 2018 nodes state a default format outside the interface */ + const char* s = _dassrc_jstr(pThis->pDoc, "format/default/mime"); + if(s == NULL) s = _dassrc_jstr(pThis->pDoc, "protocol/format/default/mime"); + if(s == NULL) s = _dassrc_jstr(pThis->pDoc, "interface/format/default/mime"); + return s ? s : ""; +} + +/* ************************************************************************* */ +/* Stream sources: translating a requisition to protocol parameters */ + +static const DasStreamSrc* _DasSrc_stream(const DasSrc* pThis) +{ + return (pThis->kind == das_src_kind_stream) ? (const DasStreamSrc*)pThis : NULL; +} + +size_t DasStreamSrc_urlCount(const DasSrc* pThis) +{ + const DasStreamSrc* pS = _DasSrc_stream(pThis); + return pS ? _dassrc_jlen(pS->pBaseUrls) : 0; +} + +const char* DasStreamSrc_baseUrl(const DasSrc* pThis, size_t uIdx) +{ + const DasStreamSrc* pS = _DasSrc_stream(pThis); + if(pS == NULL) return NULL; + const das_json_ary_el* pEl = DasJdo_aryFirst(pS->pBaseUrls); + for(size_t u = 0; (u < uIdx) && pEl; ++u) pEl = pEl->next; + return pEl ? DasJdo_string(pEl->value) : NULL; +} + +/* The output parameter list. Names are borrowed from the document or the + * requisition, values are owned because flag sets grow by appending. */ +typedef struct { const char* sName; char* sVal; } _param_t; +typedef struct { _param_t* a; size_t u; size_t uCap; } _out_t; + +static _param_t* _dassrc_outFind(_out_t* pO, const char* sName) +{ + for(size_t u = 0; u < pO->u; ++u) + if(strcmp(pO->a[u].sName, sName) == 0) return pO->a + u; + return NULL; +} + +static bool _dassrc_outSet(_out_t* pO, const char* sName, const char* sVal) +{ + _param_t* p = _dassrc_outFind(pO, sName); + if(p == NULL){ + if(pO->u == pO->uCap){ + size_t uCap = pO->uCap ? pO->uCap * 2 : 16; + _param_t* a = (_param_t*)realloc(pO->a, uCap * sizeof(_param_t)); + if(a == NULL) return false; + pO->a = a; + pO->uCap = uCap; + } + p = pO->a + pO->u; + p->sName = sName; + p->sVal = NULL; + pO->u += 1; + } + char* sNew = _dassrc_strdupz(sVal); + if(sNew == NULL) return false; + free(p->sVal); + p->sVal = sNew; + return true; +} + +static bool _dassrc_outAddFlag(_out_t* pO, const char* sName, const char* sFlagVal, const char* sSep) +{ + _param_t* p = _dassrc_outFind(pO, sName); + if((p == NULL)||(p->sVal[0] == '\0')) return _dassrc_outSet(pO, sName, sFlagVal); + char* sNew = das_string("%s%s%s", p->sVal, sSep, sFlagVal); + if(sNew == NULL) return false; + free(p->sVal); + p->sVal = sNew; + return true; +} + +static void _dassrc_outClear(_out_t* pO) +{ + for(size_t u = 0; u < pO->u; ++u) free(pO->a[u].sVal); + free(pO->a); + memset(pO, 0, sizeof(_out_t)); +} + +/* One control's value to protocol parameters, by its set block: + * + * - enum: the chosen item's "pval" if it has one, else its value, and + * an item may name a flag + * - boolean: the block fires for one state only, "value" (true unless + * said otherwise); the other state is the default and sends nothing. + * With neither pval nor flag the state itself is sent as true/false. + * - flag: the flag's value (its entry under the parameter's flags, else + * the flag name), with the flag's prefix, appended to the parameter + * with the parameter's flagSep + * - otherwise the value as given, or the block's pval + */ +static bool _DasSrcCtrl_translate(const DasSrcCtrl* pC, const char* sVal, _out_t* pO, char* sWhy, size_t uWhy) +{ + const DasSrcProp* p = pC->pProp; + char sFlag[DAS_MAX_ID_BUFSZ] = {'\0'}; + char sPVal[SCALAR_SZ] = {'\0'}; + _dassrc_strcpyz(sFlag, _dassrc_jstr(pC->pSet, "flag"), sizeof(sFlag)); + bool bHavePVal = _das_json_scalar(_dassrc_jget(pC->pSet, "pval"), sPVal, sizeof(sPVal)); + + const DasJdo* pEnum = _dassrc_jget(pC->pSet, "enum"); + if(_dassrc_jisary(pEnum)){ + char sItem[SCALAR_SZ]; + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pEnum); pEl; pEl = pEl->next){ + const DasJdo* pItem = _dassrc_jisdict(pEl->value) ? _dassrc_jget(pEl->value, "value") : pEl->value; + if(!_das_json_scalar(pItem, sItem, sizeof(sItem))) continue; + if(!_dassrc_strieq(sItem, sVal)) continue; + if(_das_json_scalar(_dassrc_jget(pEl->value, "pval"), sPVal, sizeof(sPVal))) bHavePVal = true; + else if(!bHavePVal){ _dassrc_strcpyz(sPVal, sItem, sizeof(sPVal)); bHavePVal = true; } + if(_dassrc_jstr(pEl->value, "flag")) _dassrc_strcpyz(sFlag, _dassrc_jstr(pEl->value, "flag"), sizeof(sFlag)); + break; + } + } + else if(p->type == das_srcprop_bool){ + bool bUser = false; + _dassrc_parseBool(sVal, &bUser); + const DasJdo* pTrig = _dassrc_jget(pC->pSet, "value"); + if(_dassrc_jisbool(pTrig) && (bUser != _dassrc_jtrue(pTrig))) return true; + if(!bHavePVal && (sFlag[0] == '\0')){ + _dassrc_strcpyz(sPVal, bUser ? "true" : "false", sizeof(sPVal)); + bHavePVal = true; + } + } + + if(sFlag[0] != '\0'){ + char sPath[DAS_MAX_ID_BUFSZ + 8]; + snprintf(sPath, sizeof(sPath), "flags/%s", sFlag); + const DasJdo* pF = _dassrc_jget(pC->pParam, sPath); + if(pF == NULL){ + snprintf(sWhy, uWhy, "Flag '%s' for %s is not defined under protocol parameter %s, " + "the node is broken", sFlag, p->sKey, pC->sParam); + return false; + } + char sFVal[SCALAR_SZ], sFull[SCALAR_SZ]; + if(!_das_json_scalar(_dassrc_jget(pF, "value"), sFVal, sizeof(sFVal))) _dassrc_strcpyz(sFVal, sFlag, sizeof(sFVal)); + const char* sPrefix = _dassrc_jstr(pF, "prefix"); + snprintf(sFull, sizeof(sFull), "%s%s", sPrefix ? sPrefix : "", sFVal); + const char* sSep = _dassrc_jstr(pC->pParam, "flagSep"); + if(sSep == NULL) sSep = _dassrc_jstr(pC->pParam, "flag_sep"); + if(sSep == NULL) sSep = " "; + if(!_dassrc_outAddFlag(pO, pC->sParam, sFull, sSep)){ snprintf(sWhy, uWhy, "Out of memory"); return false; } + return true; + } + if(!_dassrc_outSet(pO, pC->sParam, bHavePVal ? sPVal : sVal)){ snprintf(sWhy, uWhy, "Out of memory"); return false; } + return true; +} + +/* The value a requisition gives a property: the last good non-empty + * entry that resolved to it, derived or not */ +static const char* _dassrc_reqValueFor(const DasReq* pReq, size_t uProp) +{ + const char* sVal = NULL; + for(size_t u = 0; u < pReq->uEntry; ++u){ + const struct das_req_entry* pE = pReq->aEntry + u; + if((pE->nProp != (int)uProp)||(pE->verdict != das_req_ok)||(pE->sVal[0] == '\0')) continue; + sVal = pE->sVal; + } + return sVal; +} + +static bool _DasSrcProp_sameAsDefault(const DasSrcProp* p, const char* sVal) +{ + if(p->sValue == NULL) return false; + if(p->type == das_srcprop_bool){ + bool bA = false, bB = false; + return _dassrc_parseBool(sVal, &bA) && _dassrc_parseBool(p->sValue, &bB) && (bA == bB); + } + return _dassrc_strieq(sVal, p->sValue); +} + +size_t DasStreamSrc_url(const DasSrc* pThis, DasReq* pReq, size_t uBase, char* sBuf, size_t uLen) +{ + if(uLen > 0) sBuf[0] = '\0'; + const DasStreamSrc* pS = _DasSrc_stream(pThis); + if((pS == NULL)||(pS->pProto == NULL)||(pS->pBaseUrls == NULL)){ + daslog_error_v("Source %s has no protocol section with base URLs, it cannot be queried", + pThis->sUrl); + return 0; + } + const char* sBase = DasStreamSrc_baseUrl(pThis, uBase); + if(sBase == NULL){ + daslog_error_v("Source %s has %zu base URL(s), there is no number %zu", + pThis->sUrl, DasStreamSrc_urlCount(pThis), uBase); + return 0; + } + + /* Every reason is logged, not just the first: a blank requisition lists + the required keys up front, so the first bad entry is often "min is + required" while the user's typo sits further down */ + size_t uBad = 0; + if(DasSrc_check(pThis, pReq, &uBad) != DAS_OKAY){ + for(size_t u = 0; u < pReq->uEntry; ++u) + if(pReq->aEntry[u].verdict == das_req_bad) daslog_error_v("%s", pReq->aEntry[u].sWhy); + return 0; + } + + _out_t out = {NULL, 0, 0}; + char sWhy[1024]; + for(size_t u = 0; u < pThis->uProp; ++u){ + const DasSrcProp* p = pThis->aProp + u; + const DasSrcCtrl* pC = p->pCtrl; + if(pC == NULL) continue; + + const char* sVal = _dassrc_reqValueFor(pReq, u); + bool bDefault = false; + if(sVal == NULL){ + if(!pC->bRequired || (p->sValue == NULL)) continue; + sVal = p->sValue; + bDefault = true; + } + /* A required default fills a hole, it never replaces what another + control already put in a shared parameter */ + if(bDefault && _dassrc_outFind(&out, pC->sParam)) continue; + + /* A value equal to the default carries no information unless the + server insists on hearing it */ + if(!bDefault && !pC->bRequired && _DasSrcProp_sameAsDefault(p, sVal)) continue; + + if(!_DasSrcCtrl_translate(pC, sVal, &out, sWhy, sizeof(sWhy))){ + daslog_error_v("%s", sWhy); + _dassrc_outClear(&out); + return 0; + } + } + + for(size_t u = 0; u < pReq->uRaw; ++u){ + if(!_dassrc_outSet(&out, pReq->aRaw[u].sName, pReq->aRaw[u].sVal)){ _dassrc_outClear(&out); return 0; } + } + + /* Protocol parameters the node marks required, whatever supplies them */ + for(const das_json_dict_el* pEl = DasJdo_dictFirst(pS->pParams); pEl; pEl = pEl->next){ + if(!_dassrc_jtrue(_dassrc_jget(pEl->value, "required"))) continue; + const char* sName = pEl->name->string; + if(_dassrc_outFind(&out, sName) != NULL) continue; + + size_t n = snprintf(sWhy, sizeof(sWhy), "Required parameter %s missing, set it with", sName); + int nKeys = 0; + for(size_t i = 0; i < pThis->uProp; ++i){ + const DasSrcCtrl* pC = pThis->aProp[i].pCtrl; + if(pC && (strcmp(pC->sParam, sName) == 0)){ + n = _dassrc_catf(sWhy, sizeof(sWhy), n, "%s %s=VALUE", nKeys ? " or" : "", pThis->aProp[i].sKey); + ++nKeys; + } + } + if(nKeys == 0) _dassrc_catf(sWhy, sizeof(sWhy), n, " a raw parameter, no interface key sets it"); + daslog_error_v("%s", sWhy); + _dassrc_outClear(&out); + return 0; + } + + size_t u = _dassrc_catf(sBuf, uLen, 0, "%s", sBase); + size_t uNeed = strlen(sBase); + const char* sJoin = strchr(sBase, '?') ? "&" : "?"; + for(size_t i = 0; i < out.u; ++i){ + size_t uEnc = das_url_encode(out.a[i].sVal, NULL, 0) + 1; + char* sEnc = (char*)malloc(uEnc); + if(sEnc == NULL){ _dassrc_outClear(&out); return 0; } + das_url_encode(out.a[i].sVal, sEnc, uEnc); + u = _dassrc_catf(sBuf, uLen, u, "%s%s=%s", sJoin, out.a[i].sName, sEnc); + uNeed += strlen(sJoin) + strlen(out.a[i].sName) + 1 + strlen(sEnc); + free(sEnc); + sJoin = "&"; + } + _dassrc_outClear(&out); + return uNeed; +} + +/* ************************************************************************* */ +/* ************************************************************************* */ +/* The sources under a container */ + +size_t dassrc_loadable(const DasNode* pNode, const char** aNames, size_t uMax) +{ + if((pNode == NULL) || !DasNode_isCatalog(pNode)) return 0; + size_t uKids = DasNode_subCount(pNode); + size_t uFound = 0; + for(size_t u = 0; u < uKids; ++u){ + if(DasNode_subType(pNode, u) != d2node_stream_src) continue; + if(uFound < uMax) aNames[uFound] = DasNode_subName(pNode, u); + ++uFound; + } + return uFound; +} diff --git a/das3/source.h b/das3/source.h new file mode 100644 index 0000000..c2e42ee --- /dev/null +++ b/das3/source.h @@ -0,0 +1,659 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/** @file source.h Data sources: what they offer and how to ask for it */ + +#ifndef _das_source_h_ +#define _das_source_h_ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** @addtogroup catalog + * @{ + */ + +/** @defgroup source Data sources + * + * A DasSrc is a standalone description of one place data comes from, + * built from a DasNode or directly from JSON text. It holds a copy of + * that document, so the catalog tree it came from may be freed or + * evicted and the source keeps working. Once built it is read only and + * may be shared between threads. + * + * Three things fall out of the description: + * + * - **Properties.** The `interface` section of a source node is a + * tree of sections, groups and properties. A DasSrcProp is one leaf: + * the minimum time, the units of the frequency axis, whether the + * L-shell variable is output. Every property has a value; some also + * have a DasSrcCtrl saying how a client may change it. The fixed ones + * are not noise, they tell a client what a stream will look like + * before any of it is downloaded. + * - **Groups.** A DasSrcGroup is the box the properties sit in: a + * coordinate, a data variable, an output format, or the one anonymous + * box of server options. Forms are drawn group by group. + * - **Requisitions.** A DasSrc makes DasReq objects, blank or seeded + * from a worked example, validates them, and turns them into + * something a transport can act on. For a stream source that is a + * URL. + * + * A DasSrc resolves a requisition to a URL but never fetches from one. + * Nothing in it opens a socket, so the same code serves a command line + * tool, a browser build, or a GUI toolkit that provides the network + * functions. Facilities such as das_http_getBody() and DasIO act on the + * URL afterwards. + * + * The common path, through a DasCat: + * + * @code + * DasSrc* pSrc = DasCat_newSrc(pCat, "uiowa/juno/wav/survey"); + * DasReq* pReq = DasSrc_newReq(pSrc, 0); // seeded from example 0 + * DasReq_set(pReq, "time.res", "30"); // then adjusted + * + * char sUrl[2048]; + * if(DasStreamSrc_url(pSrc, pReq, 0, sUrl, sizeof(sUrl)) > 0) + * das_http_getBody(sUrl, ...); // or fetch(), or QNAM + * + * del_DasReq(pReq); + * del_DasSrc(pSrc); + * @endcode + * + * @anchor src_keys + * @par Keys + * + * Every property has a dotted key built from its section, group and + * property names as the node spells them: `coords.time.min`, + * `data.lshell.enabled`, `formats.csv.delimiter`, `options.ascii`. + * Options have no group, so their keys have two parts. Keys are how a + * requisition names a property, and these shorter spellings are accepted + * as well, so long as the result is unambiguous within one source: + * + * - Section names in singular or plural form, `coord` or `coords`, + * `format` or `formats`, `option` or `options`, and `fmt` or `opt`. + * - The group and property alone, `time.min`, when only one section + * has that group. + * - A group alone, `lshell` or `data.lshell`, meaning the group's + * `enabled` toggle. + * - An option alone, `ascii`, when no group has that name. + * - A bare property name, `serial`, when exactly one group has it. + * - The property names `min`, `max`, `res` and `inter` may be written + * out as `minimum`, `maximum`, `resolution` and `interval`, and + * `enable` as `enabled`. Nodes from 2018 spell them the long way, + * later nodes the short way, and users may type either. + * + * Matching is case insensitive. A spelling that fits more than one + * property is rejected with a message naming the candidates, never + * guessed. + * + * A coordinate name alone, `time`, takes range values: + * + * - `time=2021-02-21,2021-02-22` sets the minimum and maximum, and a + * third item sets the resolution or the interval, whichever the + * coordinate has. Empty items are skipped, so `time=,2021-02-22` + * sets only the maximum. + * - `time=2021-02-21` with no comma means the span that time denotes: + * the whole day; `time=2021-02-21T14` the hour from 14:00 to 15:00; + * `time=2021` the year. See dt_parsespan() for the rule. + * + * In a well formed source, the group names under `coords` are the physical + * dimension names of the emitted stream, @see DasDim. A client that has + * plotted a stream and needs more data along `time` can look up that + * group and issue a new requisition from the plot's axis range, which + * is how zoom and pan become server requests without any source specific + * code in the client. + * + * @par Drawing a form + * + * @code + * for(int sec = 0; sec < das_src_sec_size; ++sec){ + * for(size_t g = 0; g < DasSrc_groupCount(pSrc, sec); ++g){ + * const DasSrcGroup* pG = DasSrc_group(pSrc, sec, g); + * const DasSrcProp* pOn = DasSrcGroup_enable(pG); // the gate, or NULL + * + * for(size_t p = 0; p < DasSrcGroup_propCount(pG); ++p){ + * const DasSrcProp* pProp = DasSrcGroup_prop(pG, p); + * const DasSrcCtrl* pCtrl = DasSrcProp_ctrl(pProp); + * if(pCtrl == NULL) continue; // nothing to set here + * + * const char* sNow = DasReq_value(pReq, DasSrcProp_key(pProp)); + * draw_control(DasSrcProp_type(pProp), pCtrl, sNow ? sNow : DasSrcProp_value(pProp)); + * } + * } + * } + * @endcode + * + * The verdicts to mark boxes with come from DasSrc_check(), read back by + * entry through DasReq_verdict() and DasReq_why(). + * + * @par Two node layouts + * + * Source nodes from 2018 and from current servers differ in spelling and + * in where examples live. Both are read and nothing here depends on + * which one a source uses: examples from either come out as requisitions + * in interface terms. + */ + +/** The parts of a source interface, in presentation order. + * `das_src_sec_size` is the number of real sections, handy for sizing + * per section arrays. */ +typedef enum das_src_sec_enum { + das_src_sec_coord = 0, /**< "coords": time, frequency, L-shell ... */ + das_src_sec_data, /**< "data": the variables a source can emit */ + das_src_sec_format, /**< "formats": output serializations and their knobs */ + das_src_sec_option, /**< "options": server specific switches, one group */ + das_src_sec_size +} das_src_sec_e; + +/** The name of a section as it appears in keys, "coords", "data", + * "formats" or "options". "" for das_src_sec_size. */ +DAS_API const char* das_src_sec_name(das_src_sec_e sec); + +/** Kinds of source. Only stream sources are implemented today. */ +typedef enum das_src_kind_enum { + das_src_kind_inv = 0, + das_src_kind_stream, /**< node type HttpStreamSrc, see DasStreamSrc_url() */ + das_src_kind_file /**< node type FileTimeAgg, reserved */ +} das_src_kind_e; + +/** The kind of value a property holds. + * + * Nodes name types with a handful of spellings that have drifted over + * the years. They are folded into this set when a source is built. A + * type the library does not recognize is reported as text and the + * node's spelling is still available via DasSrcProp_typeName(). + */ +typedef enum das_srcprop_type_enum { + das_srcprop_text = 0, /**< any string, including unrecognized types */ + das_srcprop_bool, /**< true or false, see DasSrcCtrl_check() for spellings */ + das_srcprop_int, /**< an integer in decimal */ + das_srcprop_real, /**< a real number */ + das_srcprop_time, /**< a UTC time, ISO 8601 */ + das_srcprop_enum /**< one of a fixed list, see DasSrcCtrl_enumCount() */ +} das_srcprop_type_e; + +/** A data source description. See the group description. */ +typedef struct das_src DasSrc; + +/** A box on the form. Owned by its source. */ +typedef struct das_src_group DasSrcGroup; + +/** One property of a source, fixed or adjustable. Owned by its source. */ +typedef struct das_src_prop DasSrcProp; + +/** How an adjustable property may be changed. Owned by its property. */ +typedef struct das_src_ctrl DasSrcCtrl; + +/* ************************************************************************* */ +/* Making and identifying sources */ + +/** The children of a catalog node that new_DasSrc_node() can load + * + * Catalog paths provided by application programs usually end at a + * Collection or SourceSet whose children are the actual data sources. + * Multiple sources are listed because different clients understand how + * to fully utilize different data source types. This is the set of + * them this library interacts with, HttpStreamSrc nodes today. Which + * of several to use is the application's decision, made on whatever + * criteria it has, such as the media types each child says it provides. + * + * @param pNode A container node. A source node itself has no children + * and reports 0. A container that only lists other containers + * also reports 0. + * + * @param aNames Receives pointers to the child names, borrowed from the + * node. May be NULL with uMax 0 to just count. + * + * @param uMax The size of aNames. + * + * @returns The number of loadable children, which may exceed uMax. + */ +DAS_API size_t dassrc_loadable(const DasNode* pNode, const char** aNames, size_t uMax); + +/** Build a DasSrc from a DasNode. + * + * The node's document is copied, so the node and its tree may be freed + * afterwards. Usually reached through DasCat_newSrc(). + * + * The node must be a source itself. A container is refused with its + * loadable children named in the log, see dassrc_loadable(). + * + * @param pNode A node of type HttpStreamSrc. Borrowed. + * + * @returns A new source, to be freed with del_DasSrc(), or NULL after + * logging why the node does not describe a source. A source + * with a thin or broken interface still builds, so that it can + * at least be listed; missing protocol pieces are reported by + * whichever function builds a request from it. + */ +DAS_API DasSrc* new_DasSrc_node(const DasNode* pNode); + +/** Build a DasSrc from the JSON text of a node, with no DasNode involved. + * + * For programs that fetch documents themselves, a browser build in + * particular, and for anything else that can produce a source document. + * + * @param sDoc The JSON text, need not be NUL terminated. + * @param uLen The number of bytes in sDoc. + * @param sPathUri The catalog URI of the node, shorthands accepted. It + * is the source's identity: requisitions are stamped with it and + * later checked against it, see DasSrc_check(). NULL or empty + * leaves the source without one, and its requisitions unstamped. + * @param sUrl Where the text came from, recorded as the source's URL and + * used to resolve relative links. May be NULL. + * @returns A new source, or NULL after logging a parse problem or a + * document that is not a source node. + */ +DAS_API DasSrc* new_DasSrc_json( + const char* sDoc, size_t uLen, const char* sPathUri, const char* sUrl +); + +DAS_API void del_DasSrc(DasSrc* pThis); + +/** @memberof DasSrc */ +DAS_API das_src_kind_e DasSrc_kind(const DasSrc* pThis); + +/** The catalog URI, the source's identity, or "" for a source given none + * @memberof DasSrc */ +DAS_API const char* DasSrc_path(const DasSrc* pThis); + +/** Where the source document was loaded from, "" if unknown. @memberof DasSrc */ +DAS_API const char* DasSrc_url(const DasSrc* pThis); + +/** The node's short name, label, or "" . @memberof DasSrc */ +DAS_API const char* DasSrc_name(const DasSrc* pThis); + +/** The node's title, or "". @memberof DasSrc */ +DAS_API const char* DasSrc_title(const DasSrc* pThis); + +/** The node format version, "0.7" and so on, or "". @memberof DasSrc */ +DAS_API const char* DasSrc_version(const DasSrc* pThis); + +/** True when the protocol section says requests are authenticated. + * Servers may still ask when this is false; it is advice for the form, + * not a promise. @memberof DasSrc */ +DAS_API bool DasSrc_needsAuth(const DasSrc* pThis); + +/** The number of listed contacts for the data. @memberof DasSrc */ +DAS_API size_t DasSrc_contactCount(const DasSrc* pThis); + +/** Contact number uIdx. + * @param psEmail Receives the address, or "" if none listed. May be NULL. + * @returns The name, or NULL past the end. @memberof DasSrc */ +DAS_API const char* DasSrc_contact(const DasSrc* pThis, size_t uIdx, const char** psEmail); + +/** The source document, or an element of it by DasJdo_get() path, for + * anything the functions here do not model. NULL when the path finds + * nothing. @memberof DasSrc */ +DAS_API const DasJdo* DasSrc_jdo(const DasSrc* pThis, const char* sFragment); + +/* ************************************************************************* */ +/* Groups */ + +/** The number of groups in a section, 0 for a section the source does + * not have. Coordinates list `time` first and the rest in node order; + * the options section has one group when it has anything. + * @memberof DasSrc */ +DAS_API size_t DasSrc_groupCount(const DasSrc* pThis, das_src_sec_e sec); + +/** Group number uIdx of a section, or NULL past the end. @memberof DasSrc */ +DAS_API const DasSrcGroup* DasSrc_group(const DasSrc* pThis, das_src_sec_e sec, size_t uIdx); + +/** The group of a section with a given name, case insensitively, or + * NULL. Pass "" for the options group. @memberof DasSrc */ +DAS_API const DasSrcGroup* DasSrc_findGroup( + const DasSrc* pThis, das_src_sec_e sec, const char* sName +); + +/** @memberof DasSrcGroup */ +DAS_API das_src_sec_e DasSrcGroup_section(const DasSrcGroup* pThis); + +/** The name as the node spells it, "time", "lshell", "csv", or "" for + * the options group. @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_name(const DasSrcGroup* pThis); + +/** A short caption: the node's label, else its name. @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_label(const DasSrcGroup* pThis); + +/** A longer description, or "". @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_title(const DasSrcGroup* pThis); + +/** The property that switches the whole group on and off, or NULL. + * + * Data variables usually have one, formats always do, coordinates and + * options never. A form draws it first and greys the rest of the group + * when it is off. @memberof DasSrcGroup */ +DAS_API const DasSrcProp* DasSrcGroup_enable(const DasSrcGroup* pThis); + +/** The name of the exclusive set this group's enable toggle belongs to, + * or NULL. Formats are in one, "format": exactly one is on at a time, + * so they are radio buttons rather than check boxes. @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_xorGroup(const DasSrcGroup* pThis); + +/** The number of MIME types a format group may produce, 0 for other + * groups. @memberof DasSrcGroup */ +DAS_API size_t DasSrcGroup_mimeCount(const DasSrcGroup* pThis); + +/** MIME type number uIdx of a format group, or NULL. The first is the + * usual one. @memberof DasSrcGroup */ +DAS_API const char* DasSrcGroup_mime(const DasSrcGroup* pThis, size_t uIdx); + +/** @memberof DasSrcGroup */ +DAS_API size_t DasSrcGroup_propCount(const DasSrcGroup* pThis); + +/** Property number uIdx of the group, in node order, or NULL. + * @memberof DasSrcGroup */ +DAS_API const DasSrcProp* DasSrcGroup_prop(const DasSrcGroup* pThis, size_t uIdx); + +/** The property of a group with a given name, accepting the long and + * short spellings listed under @ref src_keys, or NULL. @memberof DasSrcGroup */ +DAS_API const DasSrcProp* DasSrcGroup_findProp(const DasSrcGroup* pThis, const char* sName); + +/** How many of the group's properties have controls. 0 means a fixed + * group, which a form may skip and a verbose listing may grey out. + * @memberof DasSrcGroup */ +DAS_API size_t DasSrcGroup_ctrlCount(const DasSrcGroup* pThis); + +/* ************************************************************************* */ +/* Properties */ + +/** The number of properties over all sections. @memberof DasSrc */ +DAS_API size_t DasSrc_propCount(const DasSrc* pThis); + +/** Property number uIdx in presentation order: sections as in + * das_src_sec_e, groups and properties within them in group order, so + * a group's properties are always adjacent. @memberof DasSrc */ +DAS_API const DasSrcProp* DasSrc_prop(const DasSrc* pThis, size_t uIdx); + +/** Find the one property a key names. + * + * Any spelling from @ref src_keys is accepted. Fixed properties are + * found too, so a caller that means to set one still has to look at + * DasSrcProp_ctrl(). A key that fits nothing or more than one property + * returns NULL; the reason is not logged, since this is also how a + * command line tool asks "is this a key", and sWhy says it instead. + * + * @param sKey The key as typed. + * @param sWhy If not NULL, receives "" on success or a message naming + * the problem and, for an ambiguous key, the candidates. + * @param uWhy The size of sWhy. + * @returns The property, or NULL. + * @memberof DasSrc + */ +DAS_API const DasSrcProp* DasSrc_findProp( + const DasSrc* pThis, const char* sKey, char* sWhy, size_t uWhy +); + +/** @memberof DasSrcProp */ +DAS_API das_src_sec_e DasSrcProp_section(const DasSrcProp* pThis); + +/** The group this property belongs to. @memberof DasSrcProp */ +DAS_API const DasSrcGroup* DasSrcProp_group(const DasSrcProp* pThis); + +/** The property name as the node spells it, "min" or "minimum", + * "enabled", "delimiter". Use DasSrcProp_key() rather than rebuilding + * keys from this. @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_name(const DasSrcProp* pThis); + +/** The full dotted key, see @ref src_keys. Unique within a source. + * @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_key(const DasSrcProp* pThis); + +/** A short human readable title. + * + * The node's title is returned when it has one. Coordinate range + * properties and enable toggles usually have none, since their meaning + * is conventional, so one is composed in sBuf from the group's label, + * "Minimum Time value to output", "Output as CSV". + * + * @param sBuf Space for a composed title, 64 bytes covers the composed + * forms. + * @param uLen The size of sBuf. + * @returns A pointer to the title, inside the source or in sBuf. Never + * NULL, but may be "" for an option with no title at all. + * @memberof DasSrcProp + */ +DAS_API const char* DasSrcProp_title(const DasSrcProp* pThis, char* sBuf, size_t uLen); + +/** The units the value is in, as the node states them, "UTC" for a time + * with no stated units, or "". Display only, values are not checked + * against units. @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_units(const DasSrcProp* pThis); + +/** @memberof DasSrcProp */ +DAS_API das_srcprop_type_e DasSrcProp_type(const DasSrcProp* pThis); + +/** The node's spelling of the type, "isotime", "boolean", or "" + * when it gave none and the type was inferred. @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_typeName(const DasSrcProp* pThis); + +/** The property's value as the node spells it, numbers in the node's + * notation. + * + * For a fixed property this is the fact: the units of an axis, the + * valid range of a coordinate. For an adjustable one it is the value in + * effect when a requisition says nothing, the default. NULL when the + * node gives null or nothing, which for a required control means the + * client must supply one. @memberof DasSrcProp */ +DAS_API const char* DasSrcProp_value(const DasSrcProp* pThis); + +/** The control for an adjustable property, NULL for a fixed one. + * @memberof DasSrcProp */ +DAS_API const DasSrcCtrl* DasSrcProp_ctrl(const DasSrcProp* pThis); + +/* ************************************************************************* */ +/* Controls */ + +/** The property this control adjusts. @memberof DasSrcCtrl */ +DAS_API const DasSrcProp* DasSrcCtrl_prop(const DasSrcCtrl* pThis); + +/** True when a requisition must supply a value or the server will refuse + * the request. A required control whose property has a value is + * satisfied by that default. @memberof DasSrcCtrl */ +DAS_API bool DasSrcCtrl_required(const DasSrcCtrl* pThis); + +/** The name of the protocol parameter a value travels in, "start_time" + * or "read.time.min". Several controls may share one, toggles that pack + * into a flag set in particular. For messages and for people composing + * requests by hand. @memberof DasSrcCtrl */ +DAS_API const char* DasSrcCtrl_param(const DasSrcCtrl* pThis); + +/** The number of allowed values of an enumerated control, 0 otherwise. + * @memberof DasSrcCtrl */ +DAS_API size_t DasSrcCtrl_enumCount(const DasSrcCtrl* pThis); + +/** Allowed value number uIdx as the node spells it, or NULL. Checks + * accept these in any letter case; what is sent is the node's spelling. + * @memberof DasSrcCtrl */ +DAS_API const char* DasSrcCtrl_enum(const DasSrcCtrl* pThis, size_t uIdx); + +/** The stated range of a numeric or time control. + * + * Nodes put ranges in several places: on the set block, on the protocol + * parameter, or for coordinate `min` and `max` on the coordinate itself + * as its valid range. All three are consulted, in that order. The high + * end of a time range is often the word "now". + * + * @param psLo Receives the low end, as the node spells it. May be NULL. + * @param psHi Receives the high end. May be NULL. + * @returns true if a range was stated, false otherwise with the outputs + * untouched. + * @memberof DasSrcCtrl + */ +DAS_API bool DasSrcCtrl_range(const DasSrcCtrl* pThis, const char** psLo, const char** psHi); + +/** Decide if a string is an acceptable value for this control. + * + * Booleans accept true, false, yes, no, on, off, 1 and 0 in any letter + * case. Enumerations accept any listed value, case insensitively. + * Integers, reals and times are checked for form only, not against the + * stated range, because servers are the authority on what they hold and + * ranges in nodes are routinely out of date. Times also accept the word + * "now". Text is always acceptable. + * + * DasSrc_check() calls this for every entry, so applications rarely + * need to, but a settings panel may want to validate as the user types. + * + * @param sWhy If not NULL, receives "" or a message naming the key, the + * value and, for an enumeration, the choices. + * @param uWhy The size of sWhy. + * @returns DAS_OKAY or DASERR_SRC. Nothing is logged. + * @memberof DasSrcCtrl + */ +DAS_API DasErrCode DasSrcCtrl_check( + const DasSrcCtrl* pThis, const char* sVal, char* sWhy, size_t uWhy +); + +/* ************************************************************************* */ +/* Requisitions */ + +/** The number of worked examples a source provides, possibly 0. + * @memberof DasSrc */ +DAS_API size_t DasSrc_exampleCount(const DasSrc* pThis); + +/** What example number uIdx shows, from its label, title or name, "" if + * it has none, NULL past the end. @memberof DasSrc */ +DAS_API const char* DasSrc_exampleLabel(const DasSrc* pThis, size_t uIdx); + +/** The argument to DasSrc_newReq() for a requisition with nothing filled in */ +#define DASSRC_BLANK_REQ -1 + +/** Make a DasReq for this source. + * + * With DASSRC_BLANK_REQ the requisition lists every required key with an + * empty value, so a form knows what must be filled and a check on it + * fails usefully. With an example number it carries that example's + * settings instead, in interface terms whatever the node layout, and the + * example's label. Either way it is stamped for this source. + * + * Nothing stops a client from making a requisition with new_DasReq() + * and never involving the source until it checks; this is the + * convenience that knows what the source wants. + * + * @returns A new requisition, to be freed with del_DasReq(), or NULL + * after logging that there is no such example. + * @memberof DasSrc + */ +DAS_API DasReq* DasSrc_newReq(const DasSrc* pThis, int nExample); + +/** Stamp a requisition as belonging to this source, see DasReq_setSrc(). + * @memberof DasSrc */ +DAS_API void DasSrc_stamp(const DasSrc* pThis, DasReq* pReq); + +/** Validate a request against this source. + * + * Every entry gets a verdict and the bad ones a reason, see DasReq_why(). + * In order: + * + * 1. A DasReq stamped with another catalog URI is refused outright + * and nothing is validated. Only the URI is compared, so a mirror + * or a local mount of the same node validates the same requisitions. + * An unstamped one is validated by whoever holds it. + * 2. Each key is resolved as DasSrc_findProp() would. No match, + * more than one match, or a fixed property is bad. A coordinate + * group name with range values, see @ref src_keys, resolves to + * derived entries for the minimum, maximum and step; the client's + * entry stays so that a form's one box stays one box. + * 3. Each value is checked with DasSrcCtrl_check(). An empty value + * is bad for a required control with no default, and otherwise + * means "not set". + * 4. Every required control not covered by an entry or a default gets + * a derived entry with an empty value and a bad verdict, so a form + * can mark a box the user never touched. + * + * Derived entries from an earlier check are dropped first, so checking + * is idempotent. Nothing is logged for bad entries, the verdicts carry + * the text; the function that builds a request from them reports them + * when it refuses. + * + * @param puBad If not NULL, receives the number of bad entries. + * @returns DAS_OKAY when every entry is good, DASERR_SRC when any is bad + * or the stamp does not match, the latter after logging. + * @memberof DasSrc + */ +DAS_API DasErrCode DasSrc_check(const DasSrc* pThis, DasReq* pReq, size_t* puBad); + +/** The MIME type a requisition is expected to produce. + * + * Taken from the enabled format group after the requisition's choices, + * or the node's default when it states one. Where a group lists several + * types the first is returned; the server's Content-Type header is the + * final word. Tells a client whether to parse the body as a das stream, + * hand it to a table viewer, or write it to disk. + * + * @returns The type, or "" when the source does not say. + * @memberof DasSrc + */ +DAS_API const char* DasSrc_mime(const DasSrc* pThis, const DasReq* pReq); + +/* ************************************************************************* */ +/* Stream sources */ + +/** The number of base URLs a stream source may be reached at, usually + * 1. More means mirrors, tried in order by a careful client. 0 for a + * source that is not a stream source or has no protocol section. + * @memberof DasSrc */ +DAS_API size_t DasStreamSrc_urlCount(const DasSrc* pThis); + +/** Base URL number uIdx, before any parameters, or NULL. Old nodes + * carry a query string in it already; DasStreamSrc_url() appends either + * way. @memberof DasSrc */ +DAS_API const char* DasStreamSrc_baseUrl(const DasSrc* pThis, size_t uIdx); + +/** The request URL for a requisition. + * + * Checks the requisition, then translates each good entry to protocol + * parameters through its control's set block: a plain value, a fixed + * protocol value, or a flag packed into a parameter shared with others + * of its set. A value equal to the default is not sent unless the + * control is required; required controls with defaults are sent even + * when the requisition is silent. Raw parameters go on last and + * override. Finally every protocol parameter the node marks required + * must be present. Values are percent encoded, see das_url_encode(). + * + * Works like snprintf: the return value is the length the whole URL + * needs, and the buffer holds as much as fits, NUL terminated. + * + * @param pReq The requisition. Its verdicts are updated as by + * DasSrc_check(). + * @param uBase Which base URL, see DasStreamSrc_urlCount(). 0 unless an + * earlier attempt failed to connect. + * @param sBuf Receives the URL. 2048 bytes is enough for any node seen + * so far, 4096 is safe. + * @param uLen The size of sBuf. + * @returns The length of the URL, or 0 after logging why none could be + * made: every bad entry, a mismatched stamp, a missing required + * parameter no key supplies, a bad base index, or a source with + * no base URLs. A URL is never empty, so 0 is unambiguous. + * @memberof DasSrc + */ +DAS_API size_t DasStreamSrc_url( + const DasSrc* pThis, DasReq* pReq, size_t uBase, char* sBuf, size_t uLen +); + +/** @} */ +/** @} */ + +#ifdef __cplusplus +} +#endif + +#endif /* _das_source_h_ */ diff --git a/das3/src_priv.h b/das3/src_priv.h new file mode 100644 index 0000000..524b923 --- /dev/null +++ b/das3/src_priv.h @@ -0,0 +1,156 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Source internals. Not installed. + * + * Everything in source.c sees these; catalog.c includes this header only + * for the default source rule. Applications see the accessors in + * source.h and nothing of the layout, so it can change after the tag. + * + * Ownership in one sentence: the source owns its document copy, its + * groups, its properties and their controls, and the example + * requisitions; everything that points into the document is borrowed + * from it and dies with it. + */ + +#ifndef _das_src_priv_h_ +#define _das_src_priv_h_ + +#include "source.h" +#include "json.h" +#include "req_priv.h" + +/* A control. Borrowed pointers point into the source's document. JSON + * number text is not NUL terminated, hence the owned copies of enum and + * range strings. */ +struct das_src_ctrl { + DasSrcProp* pProp; /* back pointer, the owner */ + const DasJdo* pSet; /* the set dict: param, flag, pval, enum, range, value */ + const DasJdo* pParam; /* the protocol parameter entry for sParam, or + NULL if the node does not describe it */ + const char* sParam; /* borrowed from pSet, never NULL */ + bool bRequired; /* set/required or the parameter's required */ + char** aEnum; /* owned, enum values in node order */ + size_t uEnum; + char* sLo; /* owned, range ends, NULL when none stated */ + char* sHi; +}; + +struct das_src_prop { + das_src_sec_e sec; + DasSrcGroup* pGroup; /* back pointer */ + const char* sName; /* borrowed dict key, the node's spelling */ + char* sKey; /* owned, "coords.time.min" */ + const DasJdo* pJdo; /* the property dict: value, set, type, label... */ + das_srcprop_type_e type; + const char* sTypeName; /* borrowed, "" when inferred */ + char* sValue; /* owned scalar copy, NULL when null or absent */ + DasSrcCtrl* pCtrl; /* owned, NULL for a fixed property */ +}; + +/* Properties of a group are adjacent in the source's aProp array, so a + * group is an index range. */ +struct das_src_group { + const DasSrc* pSrc; /* back pointer, for reaching the properties */ + das_src_sec_e sec; + const char* sName; /* borrowed dict key, "" for options */ + const DasJdo* pJdo; /* the coordinate, variable, format or options dict */ + size_t uFirst; /* into DasSrc.aProp */ + size_t uCount; + int nEnable; /* index of the "enabled" prop, -1 if none */ + const char* sXor; /* borrowed, enable prop's xorGroup, or NULL */ + const char** aMime; /* owned array of borrowed strings, formats only */ + size_t uMime; +}; + +struct das_src { + das_src_kind_e kind; + char* sPath; /* owned, "" when none */ + char* sUrl; /* owned, "" when none */ + + DasJdo* pDoc; /* owned, the parsed copy of the node document */ + const DasJdo* pIface; /* "interface", may be NULL */ + bool bLegacy; /* the 2018 layout, see _DasSrc_isLegacy() */ + + DasSrcGroup* aGroup; /* owned, in section then presentation order */ + size_t uGroup; + size_t aSecFirst[das_src_sec_size]; /* first group of each section */ + size_t aSecCount[das_src_sec_size]; + + DasSrcProp* aProp; /* owned, grouped as the groups say */ + size_t uProp; + DasSrcCtrl* aCtrl; /* owned, one per settable property, in + property order; props point into it */ + size_t uCtrl; + + DasReq** aExample; /* owned, one per example, built at load */ + size_t uExample; +}; + +/* The stream source. Base first so a DasStreamSrc* is a DasSrc*; the cast + * back is made only after kind says das_src_kind_stream. */ +typedef struct das_stream_src { + DasSrc base; + const DasJdo* pProto; /* "protocol", NULL makes the source unqueryable */ + const DasJdo* pParams; /* protocol "httpParams" or "http_params" */ + const DasJdo* pBaseUrls; /* protocol "baseUrls" or "base_urls" array */ +} DasStreamSrc; + +/* Key grammar helpers, see "Keys" in source.h. They live next to the + * parser so that the spellings accepted on input and the spellings + * written into keys cannot drift apart. */ + +/* Fold a section name as a user may type it to a section. Case + * insensitive. Returns false if the word is not a section name. */ +bool _das_src_secByName(const char* sName, das_src_sec_e* pSec); + +/* The short spelling of a property name: "minimum" -> "min", "enable" -> + * "enabled". Names with no alias are returned as given. */ +const char* _das_src_propName(const char* sProp); + +/* True if two property names are the same after _das_src_propName() and + * case folding. */ +bool _das_src_propEq(const char* sA, const char* sB); + +/* Split a dotted key into at most three parts. + * + * @param aParts Receives the parts, each NUL terminated. + * @returns The number of parts, 1 to 3, or 0 if the key is empty, has an + * empty part, has more than three parts, or has a part that does + * not fit. */ +int _das_src_splitKey(const char* sKey, char aParts[3][DAS_MAX_ID_BUFSZ]); + +/* Find properties matching a key, the engine behind DasSrc_findProp() + * and DasSrc_check(). + * + * Tries the spellings from most to least specific: SEC.GROUP.PROP; + * SEC.PROP for options or SEC.GROUP for a toggle; GROUP.PROP in any + * section; a lone option name or group toggle; and last a bare property + * name. The first level that matches anything decides, so a lone name + * that is both an option and a property reports the option. + * + * @param aOut Receives up to nMax matches. May be NULL with nMax 0 to + * count only. + * @returns The number of matches, which may exceed nMax. 0 is no such + * key, 1 a hit, more is ambiguous and aOut holds the candidates. */ +int _DasSrc_match(const DasSrc* pThis, const char* sKey, const DasSrcProp** aOut, int nMax); + +/* If sKey, with or without a section prefix, names a coordinate group, + * return that group, else NULL. How a check recognizes the range forms. */ +const DasSrcGroup* _DasSrc_coordGroup(const DasSrc* pThis, const char* sKey); + +#endif /* _das_src_priv_h_ */ diff --git a/das3/time.c b/das3/time.c index e1396f7..02e1778 100644 --- a/das3/time.c +++ b/das3/time.c @@ -1,4 +1,4 @@ -/* Copyright (C) 2015-2024 Chris Piker +/* Copyright (C) 2015-2026 Chris Piker * Copyright (C) 1993-1998 Larry Granroth * * This file used to be named parsetime.c. It is part of das2C, the Core @@ -389,6 +389,106 @@ bool dt_parsetime(das_time* dt, const char* sTime) else return false; } +/* The span rule needs to know which field was the last one written, and + * parsetime() does not say, so the string is scanned for its shape + * first and parsetime() is trusted for the values. Shapes handled: + * Y, Y-M, Y-M-D, Y-DOY, each optionally followed by T or space and + * h, h:m, h:m:s, with an optional trailing Z. Anything else, including + * fractional seconds, is refused. */ +bool dt_parsespan(const char* sTime, das_time* pBeg, das_time* pEnd) +{ + if(sTime == NULL) return false; + char sBuf[64]; + strncpy(sBuf, sTime, sizeof(sBuf) - 1); + sBuf[sizeof(sBuf) - 1] = '\0'; + + /* trim blanks and a trailing Z */ + char* pS = sBuf; + while(isspace((unsigned char)*pS)) ++pS; + size_t uLen = strlen(pS); + while((uLen > 0) && isspace((unsigned char)pS[uLen - 1])) pS[--uLen] = '\0'; + if((uLen > 0) && ((pS[uLen - 1] == 'Z')||(pS[uLen - 1] == 'z'))) pS[--uLen] = '\0'; + if(uLen == 0) return false; + + /* the date and the time of day */ + char* pTod = strchr(pS, 'T'); + if(pTod == NULL) pTod = strchr(pS, ' '); + if(pTod != NULL){ *pTod = '\0'; ++pTod; if(*pTod == '\0') return false; } + + /* count fields and insist on digits */ + int nDate = 0, aDateLen[3] = {0, 0, 0}; + for(const char* p = pS; ; ++p){ + if((*p == '-')||(*p == '\0')){ + if(nDate == 3) return false; + if(aDateLen[nDate] == 0) return false; + ++nDate; + if(*p == '\0') break; + } + else if(isdigit((unsigned char)*p)) aDateLen[nDate] += 1; + else return false; + } + int nTod = 0; + if(pTod != NULL){ + int nLen = 0; + for(const char* p = pTod; ; ++p){ + if((*p == ':')||(*p == '\0')){ + if((nTod == 3)||(nLen == 0)) return false; + ++nTod; nLen = 0; + if(*p == '\0') break; + } + else if(isdigit((unsigned char)*p)) ++nLen; + else return false; /* a '.' lands here: fractions have no span */ + } + } + + /* which unit steps: 0 year, 1 month, 2 day, 3 hour, 4 minute, 5 second */ + int nUnit; + bool bDoy = (nDate == 2) && (aDateLen[1] == 3); + if(nTod > 0){ + if((nDate < 3) && !bDoy) return false; /* a time of day needs a whole date */ + nUnit = 2 + nTod; + } + else{ + nUnit = bDoy ? 2 : nDate - 1; + } + + das_time dt; + dt_null(&dt); + if(nUnit == 0){ + /* parsetime() refuses a bare year */ + dt.year = atoi(pS); + dt.month = 1; dt.mday = 1; + if(dt.year <= 0) return false; + } + else if(nUnit == 1){ + dt.year = atoi(pS); + dt.month = atoi(strchr(pS, '-') + 1); + dt.mday = 1; + if((dt.year <= 0)||(dt.month < 1)||(dt.month > 12)) return false; + } + else{ + if(pTod != NULL) *(pTod - 1) = 'T'; /* put the string back together */ + if(!dt_parsetime(&dt, pS)) return false; + } + dt_tnorm(&dt); + + das_time dtEnd; + dt_copy(&dtEnd, &dt); + switch(nUnit){ + case 0: dtEnd.year += 1; break; + case 1: dtEnd.month += 1; break; + case 2: dtEnd.mday += 1; break; + case 3: dtEnd.hour += 1; break; + case 4: dtEnd.minute += 1; break; + default: dtEnd.second += 1.0; break; + } + dt_tnorm(&dtEnd); + + if(pBeg) dt_copy(pBeg, &dt); + if(pEnd) dt_copy(pEnd, &dtEnd); + return true; +} + /* ************************************************************************* */ bool dt_now(das_time* pDt) { diff --git a/das3/time.h b/das3/time.h index d227c9f..768886e 100644 --- a/das3/time.h +++ b/das3/time.h @@ -1,5 +1,5 @@ -/* Copyright (C) 1997-2017 Larry Granroth - * Chris Piker +/* Copyright (C) 1997 Larry Granroth + * 2012-2026 Chris Piker * * This file is part of das2C, the Core Das2 C Library. * @@ -127,6 +127,26 @@ DAS_API void dt_null(das_time* pDt); */ DAS_API bool dt_parsetime(das_time* pDt, const char* sTime); +/** Parse a time string as the span it denotes. + * + * A time written to limited precision names an interval, not an instant: + * "2026-01-02" is the whole of January 2nd, "2026-01-02T14" the hour from + * 14:00 to 15:00, "2026" the year. This reads the string the way + * dt_parsetime() does and also sets pEnd to the start of the next unit + * of the least significant field given. Day of year forms, "2026-002", + * step the day. A string with a fraction of a second, "T14:30:15.5", + * denotes no natural span and is refused, as is anything dt_parsetime() + * refuses. + * + * @param sTime An ISO 8601 date or date and time, year through whole + * seconds, with or without a trailing Z. + * @param pBeg Receives the start of the span. May be NULL. + * @param pEnd Receives the end of the span, exclusive. May be NULL. + * @returns true if the string was understood, false otherwise with both + * outputs untouched. + */ +DAS_API bool dt_parsespan(const char* sTime, das_time* pBeg, das_time* pEnd); + /** Initialize a das_time to the current UTC time. * diff --git a/das3/util.c b/das3/util.c index b91a141..62bc291 100644 --- a/das3/util.c +++ b/das3/util.c @@ -65,6 +65,7 @@ static das_error_msg* g_msgBuf = NULL; #define HOME_DIR_SZ 256 static char g_sHome[HOME_DIR_SZ] = {'\0'}; +static char g_sUserDir[HOME_DIR_SZ] = {'\0'}; /* Locale handling */ #ifdef _WIN32 @@ -196,6 +197,13 @@ void das_init( else strcpy(g_sHome, "/tmp"); #endif + + /* Everything das keeps for a user lives under one directory, ~/.das on + * every OS the way ssh and git do it, unless DAS_USERDIR says otherwise. + * Read once here so a setenv() later in the process cannot move the + * cache or the credentials file out from under an open session. A + * program that wants to choose calls das_set_userdir(). */ + das_set_userdir(NULL); } void das_finish(){ @@ -716,6 +724,23 @@ const char* das_userhome(void) return g_sHome; } +const char* das_userdir(void) +{ + return g_sUserDir; +} + +void das_set_userdir(const char* sDir) +{ + if((sDir == NULL)||(sDir[0] == '\0')) sDir = getenv("DAS_USERDIR"); + if((sDir != NULL) && (sDir[0] != '\0')){ + strncpy(g_sUserDir, sDir, HOME_DIR_SZ - 1); + g_sUserDir[HOME_DIR_SZ - 1] = '\0'; + } + else{ + snprintf(g_sUserDir, HOME_DIR_SZ, "%s" DAS_DSEPS ".das", g_sHome); + } +} + bool das_isfile(const char* path) { struct stat stbuf; @@ -725,6 +750,22 @@ bool das_isfile(const char* path) else return false; } +int das_rename(const char* sFrom, const char* sTo) +{ +#ifdef _WIN32 + if(MoveFileExA(sFrom, sTo, MOVEFILE_REPLACE_EXISTING)) return 0; + /* The two causes a caller is likely to act on, the rest is EIO */ + switch(GetLastError()){ + case ERROR_FILE_NOT_FOUND: case ERROR_PATH_NOT_FOUND: errno = ENOENT; break; + case ERROR_ACCESS_DENIED: case ERROR_SHARING_VIOLATION: errno = EACCES; break; + default: errno = EIO; + } + return -1; +#else + return rename(sFrom, sTo); +#endif +} + void das_stdio_binary(FILE* pFile) { #ifdef _WIN32 @@ -734,6 +775,19 @@ void das_stdio_binary(FILE* pFile) #endif } +/* Length of a leading drive spec, such as "C:", that's never made as a + directory. stat("C:") doesn't report a directory, so mkdir would be + tried and fail. */ +static size_t _das_drivelen(const char* sPath) +{ +#ifdef WIN32 + if((sPath[0] != '\0') && (sPath[1] == ':')) return 2; +#else + (void)sPath; +#endif + return 0; +} + #ifndef WIN32 bool das_copyfile(const char* src, const char* dest, mode_t mode) #else @@ -775,9 +829,11 @@ bool das_copyfile(const char* src, const char* dest) strncpy(sPath, dest, 255); uLen = strlen(sPath); - + size_t uDrive = _das_drivelen(sPath); + for(u = 0; u0) && (u < uLen-1)){ + if(((sPath[u] == '/')||(sPath[u] == DAS_DSEPC)) && (u>uDrive) && (u < uLen-1)){ + char cSep = sPath[u]; sPath[u] = '\0'; if(! das_isdir(sPath) ){ @@ -797,7 +853,7 @@ bool das_copyfile(const char* src, const char* dest) } } - sPath[u] = '/'; + sPath[u] = cSep; } } @@ -845,10 +901,12 @@ DasErrCode das_mkdirsto(const char* path) #endif size_t uLen = strlen(sPath); + size_t uDrive = _das_drivelen(sPath); int nErr; for(size_t u = 0; u < uLen; u++){ - if((sPath[u] == DAS_DSEPC) && (u>0) && (u < uLen-1)){ + if(((sPath[u] == '/')||(sPath[u] == DAS_DSEPC)) && (u>uDrive) && (u < uLen-1)){ + char cSep = sPath[u]; sPath[u] = '\0'; if(! das_isdir(sPath) ){ @@ -866,7 +924,7 @@ DasErrCode das_mkdirsto(const char* path) } } - sPath[u] = '/'; + sPath[u] = cSep; } } return DAS_OKAY; @@ -1179,3 +1237,55 @@ const char* das_stream_type_toStr(das_stream_type type) default: return "unknown"; } } + +/* ************************************************************************* */ +/* Strings that name places: URLs, files and catalog URIs */ + +size_t das_url_encode(const char* sIn, char* sOut, size_t uOut) +{ + static const char* sHex = "0123456789ABCDEF"; + size_t uNeed = 0; + if(uOut > 0) sOut[0] = '\0'; + for(const unsigned char* p = (const unsigned char*)sIn; p && *p; ++p){ + bool bPlain = isalnum(*p)||(*p == '-')||(*p == '.')||(*p == '_')||(*p == '~'); + size_t n = bPlain ? 1 : 3; + if(uNeed + n < uOut){ + if(bPlain){ + sOut[uNeed] = (char)*p; + } + else{ + sOut[uNeed] = '%'; + sOut[uNeed + 1] = sHex[*p >> 4]; + sOut[uNeed + 2] = sHex[*p & 0x0F]; + } + sOut[uNeed + n] = '\0'; + } + uNeed += n; + } + return uNeed; +} + +static bool _das_hasPfx(const char* s, const char* sPrefix) +{ + return (s != NULL) && (strncmp(s, sPrefix, strlen(sPrefix)) == 0); +} + +bool das_isUri(const char* sPath) +{ + if((sPath == NULL) || !isalpha((unsigned char)sPath[0])) return false; + const char* p = sPath + 1; + while(isalnum((unsigned char)*p) || (*p == '+') || (*p == '-') || (*p == '.')) ++p; + /* A single letter before the colon is a Windows drive, not a scheme */ + return (*p == ':') && (p > sPath + 1); +} + +bool das_isUrlOrFile(const char* sPath) +{ + if(sPath == NULL) return false; + if(_das_hasPfx(sPath, "http://")||_das_hasPfx(sPath, "https://")||_das_hasPfx(sPath, "file://")) return true; + if((sPath[0] == '.')||(sPath[0] == '/')||(sPath[0] == '\\')) return true; + if(isalpha((unsigned char)sPath[0]) && (sPath[1] == ':') && ((sPath[2] == '\\')||(sPath[2] == '/'))) + return true; /* a Windows drive, catalog URIs never have their first colon at index 1 */ + size_t u = strlen(sPath); + return (u > 5) && (strcmp(sPath + u - 5, ".json") == 0); +} diff --git a/das3/util.h b/das3/util.h index d8b7fff..d913fd7 100644 --- a/das3/util.h +++ b/das3/util.h @@ -183,6 +183,8 @@ DAS_API DasErrCode das_error_func_fixed( * - @b 42 : uri.c - DASERR_URI * - @b 43 : form*.c - DASERR_FORM * - @b 44 : generator.c - DASERR_GEN + * - @b 45 : catalog.c - DASERR_CAT + * - @b 46 : settable.c - DASERR_SRC * * Application programs are recommended to use values 64 and above to avoid * colliding with future das2 error codes. @@ -440,6 +442,44 @@ DAS_API const char* das_xml_escape(char* dest, const char* src, size_t uOutLen); */ DAS_API char* das_strdup(const char* sIn); +/** Percent encode a string for use in a URL query. + * + * Everything but the RFC 3986 unreserved characters, letters, digits and + * "-._~", is encoded, including the ':' in time strings. Servers decode + * before use so over-encoding is harmless, and it is what most encoders + * do. Works like snprintf: the output is always NUL terminated when uOut + * is at least 1, and the return value says how much room was needed. + * + * @param sIn The text to encode. + * @param sOut Receives the encoding. Three times the input length plus + * one always suffices. + * @param uOut The size of sOut. + * @returns The length of the full encoding, not counting the terminator. + * If this is uOut or more the output was cut short. + */ +DAS_API size_t das_url_encode(const char* sIn, char* sOut, size_t uOut); + +/** Is this string a node location rather than a catalog path? + * + * True for http://, https:// and file:// URLs, for anything starting with + * '.', '/', '\' or a drive letter, and for anything ending in ".json". All else + * is taken to be a catalog path or URI. This is the rule DasCat_resolve() + * uses, published so command line parsers can agree with it. + * + */ +DAS_API bool das_isUrlOrFile(const char* sPath); + +/** Is this string a full URI rather than a relative catalog path? + * + * True when it starts with a URI scheme, a letter followed by letters, + * digits, '+', '-' or '.' and then a colon, as in "tag:", "doi:" and the + * shorthands "site:" and "test:". A single letter before the colon is a + * Windows drive, not a scheme. Note that URLs are URIs too; test with + * das_isUrlOrFile() first when the difference matters. + * + */ +DAS_API bool das_isUri(const char* sPath); + /** A memset that handles multi-byte items * * Uses memcpy because the amount of data written in each call goes up @@ -503,6 +543,33 @@ DAS_API DasErrCode das_mkdirsto(const char* path); */ DAS_API const char* das_userhome(void); +/** Get the directory for everything das keeps on behalf of the current user + * + * This is ".das" under the home directory on every platform, the way ssh + * and git keep their files, or the value of the DAS_USERDIR environment + * variable if it was set when das_init() ran, or whatever the program gave + * das_setUserDir(). The catalog node cache and saved credentials live + * under it. Only the path is returned, nothing is created; whatever + * writes under it makes the directories it needs. + * + * @return A pointer to a global string, never NULL or empty. + */ +DAS_API const char* das_userdir(void); + +/** Choose the directory das_userdir() reports + * + * For programs that embed the library and want to set the location + * themselves instead of relying on the environment. Call it right after + * das_init() and before any DasCat or other object that reads the + * directory is made: those take their copy when constructed and are not + * moved by a later call. + * + * @param sDir The directory to use, copied. NULL or an empty string goes + * back to the default, the DAS_USERDIR environment variable if set, + * else ".das" under the home directory. + */ +DAS_API void das_set_userdir(const char* sDir); + /** Copy a file to a destination creating directories as needed. * * If the files exists at the destination it in overwritten. Directories are @@ -532,6 +599,20 @@ DAS_API bool das_copyfile(const char* src, const char* dest, mode_t mode); */ DAS_API bool das_isfile(const char* path); +/** Rename a file, replacing the destination if it exists. + * + * POSIX rename(2) already does this, atomically. The Windows C runtime's + * rename() fails when the destination exists, so there this is a wrapper + * around MoveFileEx() with MOVEFILE_REPLACE_EXISTING, which is the atomic + * replace within one volume. Use it for the write-to-side-file-then-rename + * pattern so that the pattern behaves the same everywhere. + * + * @param sFrom The existing file + * @param sTo The new name, any file already there is replaced + * @return 0 on success, -1 on failure with errno set as rename(2) would + */ +DAS_API int das_rename(const char* sFrom, const char* sTo); + /** Put a stdio stream in binary mode * * Das streams count bytes, but on Windows stdin and stdout start in text mode, diff --git a/test/TestCat.c b/test/TestCat.c new file mode 100644 index 0000000..c86558b --- /dev/null +++ b/test/TestCat.c @@ -0,0 +1,455 @@ +/** @file TestCat.c Unit tests for DasCat, the catalog session, over local + * node files. No network. */ + +/* Author: Chris Piker , via Claude Fable 5.1 + * + * This file is intended to demonstrate an interface. This is free + * and unencumbered software released into the public domain + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this file, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this file dedicate any and all copyright interest in this file to + * the public domain. We make this dedication for the benefit of the + * public at large and to the detriment of our heirs and successors. We + * intend this dedication to be an overt act of relinquishment in + * perpetuity of all present and future rights to this file under + * copyright law. + * + * THIS FILE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * + * For more information, please refer to + */ + +/* Runs from the top of the source tree, the fixture node files in + * test/das_fetch/cat/ refer to each other by paths from there. The tree + * is site -> uiowa -> geomag -> flex, with two children under uiowa that + * can be listed but not loaded. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#endif + +#include +#include + +#define SITE "test/das_fetch/cat/site.json" +#define UIOWA "test/das_fetch/cat/uiowa.json" +#define UIOWA_URI "tag:das2.org,2012:site:/uiowa" + +/* A user directory that does not exist. Nothing writes there: the + fixtures are local files, which are never cached. */ +#define TEST_USERDIR "test_das_userdir_never_created" + + +static int g_nFail = 0; +static int g_nPass = 0; + +#define CHECK(cond, ...) do{ \ + if(cond){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } \ +}while(0) + +static bool streq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return (a == b); + return strcmp(a, b) == 0; +} + +/* Refusals log at error level by design; keep expected ones out of the + * test output */ +static DasErrCode quiet(DasErrCode (*f)(DasCat*, const char*), DasCat* p, const char* s) +{ + daslog_setlevel(DASLOG_CRIT); + DasErrCode n = f(p, s); + daslog_setlevel(DASLOG_WARN); + return n; +} +/* for the two-string setters, argument order is (url, uri) as in catalog.h */ +static DasErrCode quiet2( + DasErrCode (*f)(DasCat*, const char*, const char*), DasCat* p, const char* a, const char* b +){ + daslog_setlevel(DASLOG_CRIT); + DasErrCode n = f(p, a, b); + daslog_setlevel(DASLOG_WARN); + return n; +} +static DasNode* quietResolve(DasCat* p, const char* s) +{ + daslog_setlevel(DASLOG_CRIT); + DasNode* pNode = DasCat_resolve(p, s); + daslog_setlevel(DASLOG_WARN); + return pNode; +} + +/* ************************************************************************* */ +static void test_isUrlOrFile(void) +{ + printf("TEST: das_isUrlOrFile\n"); + CHECK(das_isUrlOrFile("http://a/b.json"), "http"); + CHECK(das_isUrlOrFile("https://a/b"), "https"); + CHECK(das_isUrlOrFile("file:///x/y"), "file scheme"); + CHECK(das_isUrlOrFile("./node.json"), "dot path"); + CHECK(das_isUrlOrFile("/abs/node"), "absolute path"); + CHECK(das_isUrlOrFile("C:\\nodes\\x"), "windows drive path by backslash"); + CHECK(das_isUrlOrFile("test/das_fetch/cat/site.json"), "relative path ending in .json"); + CHECK(!das_isUrlOrFile("uiowa/juno/wav/survey"), "catalog path"); + CHECK(!das_isUrlOrFile("tag:das2.org,2012:site:/uiowa"), "catalog URI"); + CHECK(!das_isUrlOrFile("site:/uiowa"), "shorthand URI"); + CHECK(!das_isUrlOrFile(NULL), "NULL"); + CHECK(!das_isUrlOrFile(""), "empty"); + + printf("TEST: das_isUri\n"); + CHECK(das_isUri("tag:das2.org,2012:site:/uiowa"), "tag URI"); + CHECK(das_isUri("doi:10.1037/a0040251/"), "doi URI"); + CHECK(das_isUri("site:/uiowa") && das_isUri("test:x"), "shorthands"); + CHECK(das_isUri("https://host/x.json"), "a URL is a URI too"); + CHECK(!das_isUri("uiowa/juno/wav:x"), "relative path, colon after a slash"); + CHECK(!das_isUri("C:\\nodes\\x") && !das_isUri("C:/x"), "drive letters are not schemes"); + CHECK(!das_isUri("1tag:x") && !das_isUri(":x") && !das_isUri("") && !das_isUri(NULL), "not schemes"); +} + +/* ************************************************************************* */ +static void test_policy(void) +{ + printf("TEST: prefix, seed and mount setters\n"); + DasCat* pCat = new_DasCat(NULL, "TestCat"); + CHECK(pCat != NULL, "new_DasCat"); + CHECK(DasCat_credMngr(pCat) == NULL, "no manager"); + CHECK(streq(DasCat_agent(pCat), "TestCat"), "agent copied"); + CHECK(streq(DasCat_prefix(pCat), DASCAT_DEF_PREFIX), "default prefix"); + CHECK(!DasCat_hasSeed(pCat), "no seed yet"); + CHECK(DasCat_seedUrl(pCat) == NULL, "no seed url"); + CHECK(streq(DasCat_prefix(pCat), DASCAT_DEF_PREFIX), "no seed: paths start at the default prefix"); + CHECK(DasCat_mountCount(pCat) == 0, "no mounts"); + CHECK(DasCat_lastUri(pCat) == NULL, "no resolve yet"); + + CHECK(DasCat_setPrefix(pCat, "site:/uiowa") == DAS_OKAY, "prefix shorthand"); + CHECK(streq(DasCat_prefix(pCat), UIOWA_URI), "prefix expanded: %s", DasCat_prefix(pCat)); + CHECK(DasCat_setPrefix(pCat, "site:uiowa") == DAS_OKAY, "prefix shorthand, slash left out"); + CHECK(streq(DasCat_prefix(pCat), UIOWA_URI), "slash supplied: %s", DasCat_prefix(pCat)); + CHECK(DasCat_setPrefix(pCat, "test:") == DAS_OKAY, "bare shorthand"); + CHECK(streq(DasCat_prefix(pCat), "tag:das2.org,2012:test:/"), "bare shorthand ends in a slash"); + CHECK(DasCat_setPrefix(pCat, "tag:das2.org,2012:test:/") == DAS_OKAY, "prefix full"); + CHECK(quiet(DasCat_setPrefix, pCat, SITE) != DAS_OKAY, "a file is not a prefix"); + CHECK(quiet(DasCat_setPrefix, pCat, "") != DAS_OKAY, "empty prefix"); + + CHECK(quiet2(DasCat_setSeed, pCat, "uiowa", NULL) != DAS_OKAY, "a path is not a seed"); + CHECK(DasCat_setSeed(pCat, SITE, NULL) == DAS_OKAY, "seed by file"); + CHECK(DasCat_hasSeed(pCat), "has seed"); + CHECK(streq(DasCat_seedUrl(pCat), SITE), "seed url kept"); + CHECK(quiet2(DasCat_setSeed, pCat, UIOWA, NULL) != DAS_OKAY, "one seed only"); + CHECK(quiet2(DasCat_setSeed, pCat, UIOWA, UIOWA_URI) != DAS_OKAY, "no second seed with a prefix either"); + CHECK(quiet(DasCat_setPrefix, pCat, "site:/") != DAS_OKAY, "no prefix once a seed is set, paths start at the seed"); + CHECK(streq(DasCat_prefix(pCat), ""), "the seed superseded the prefix set before it"); + CHECK(streq(DasCat_prefix(pCat), ""), "a seed without a prefix is a root, paths start at nothing"); + CHECK(DasCat_mount(pCat, UIOWA, UIOWA_URI) == DAS_OKAY, "a mount joins an offline catalog"); + CHECK(DasCat_mountCount(pCat) == 1, "the seed is not a mount"); + CHECK(DasCat_covers(pCat, UIOWA_URI "/x") && !DasCat_covers(pCat, "tag:das2.org,2012:site:/swri"), + "covered by the mount, walled off elsewhere"); + del_DasCat(pCat); + + pCat = new_DasCat(NULL, NULL); + CHECK(DasCat_agent(pCat) == NULL, "NULL agent stays NULL"); + CHECK(quiet2(DasCat_mount, pCat, UIOWA, "") != DAS_OKAY, "a mount needs a URI"); + CHECK(quiet2(DasCat_setSeed, pCat, UIOWA, UIOWA) != DAS_OKAY, "seed prefix can't be a file"); + CHECK(quiet2(DasCat_setSeed, pCat, "uiowa", UIOWA_URI) != DAS_OKAY, "seed URL must be a location"); + CHECK(DasCat_setSeed(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "seed with a prefix"); + CHECK(streq(DasCat_prefix(pCat), UIOWA_URI), "seed prefix expanded, and it is where paths start"); + CHECK(DasCat_hasSeed(pCat), "a seed with a prefix is a seed"); + CHECK(quiet2(DasCat_setSeed, pCat, SITE, NULL) != DAS_OKAY, "no second seed"); + del_DasCat(pCat); + + pCat = new_DasCat(NULL, NULL); + CHECK(DasCat_mount(pCat, UIOWA, UIOWA_URI) == DAS_OKAY, "mount one"); + CHECK(DasCat_mount(pCat, SITE, "site:swri") == DAS_OKAY, "mount two, slash left out"); + CHECK(DasCat_mountCount(pCat) == 2, "two mounts"); + CHECK(!DasCat_hasSeed(pCat), "mounts are not a seed"); + CHECK(DasCat_covers(pCat, "tag:das2.org,2012:site:/swri/x"), "slash supplied in the mount URI"); + CHECK(DasCat_covers(pCat, "tag:das2.org,2012:site:/lanl"), "mounts fall through"); + CHECK(DasCat_setSeed(pCat, SITE, "test:/") == DAS_OKAY, "a seed after mounts isolates them"); + CHECK(DasCat_hasSeed(pCat) && (DasCat_mountCount(pCat) == 2), "two mounts, the seed is not one"); + CHECK(!DasCat_covers(pCat, "tag:das2.org,2012:site:/lanl"), "no fall through now"); + del_DasCat(pCat); +} + +/* ************************************************************************* */ +static void test_seed(void) +{ + printf("TEST: a node file as the seed\n"); + DasCat* pCat = new_DasCat(NULL, "TestCat"); + DasCat_setCache(pCat, NULL, 0); /* explicitly off, file nodes never cache anyway */ + CHECK(DasCat_setSeed(pCat, SITE, NULL) == DAS_OKAY, "set seed"); + + DasNode* pRoot = DasCat_resolve(pCat, NULL); + CHECK(pRoot != NULL, "root resolves"); + CHECK(pRoot && DasNode_isCatalog(pRoot), "root is a catalog"); + CHECK(streq(DasCat_lastUri(pCat), ""), "no path: last URI is empty, a root has none"); + CHECK(pRoot && (DasNode_pathUri(pRoot) == NULL), "the node has no URI"); + CHECK(DasCat_seed(pCat) == pRoot, "the seed accessor is the same node"); + CHECK(DasCat_resolve(pCat, "") == pRoot, "empty path is the same as NULL"); + + /* Policy is frozen now */ + CHECK(quiet(DasCat_setPrefix, pCat, "site:/") != DAS_OKAY, "no policy change after resolve"); + + DasNode* pSrc = DasCat_resolve(pCat, "uiowa/geomag/flex"); + CHECK(pSrc != NULL, "deep path resolves"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "leaf is a stream source"); + char sExpect[1100]; + /* Below a root, names are the path from it: the root's ":/" + separator is not used since there is nothing to join to */ + strcpy(sExpect, "uiowa/geomag/flex"); + CHECK(streq(DasCat_lastUri(pCat), sExpect), "relative path is the name below a root: %s", + DasCat_lastUri(pCat)); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), sExpect), "node path URI: %s", + pSrc ? DasNode_pathUri(pSrc) : "(null)"); + CHECK(DasCat_resolve(pCat, "uiowa/geomag/flex") == pSrc, "second resolve is the cached node"); + + DasNode* pUiowa = DasCat_resolve(pCat, "uiowa"); + CHECK(pUiowa && DasNode_isCatalog(pUiowa), "intermediate node"); + CHECK(pUiowa && (DasNode_subCount(pUiowa) == 3), "uiowa lists three children"); + CHECK(pUiowa && (DasNode_subType(pUiowa, 2) == d2node_inv), "unknown child type is d2node_inv"); + + /* Misses, and what the catalog can say about them */ + CHECK(quietResolve(pCat, "uiowa/nope/deeper") == NULL, "bad child fails"); + strcpy(sExpect, "uiowa/nope/deeper"); + CHECK(streq(DasCat_lastUri(pCat), sExpect), "last URI is the failed path"); + CHECK(DasCat_covers(pCat, DasCat_lastUri(pCat)), "under the root is covered"); + CHECK(!DasCat_covers(pCat, UIOWA_URI), "a URI with a scheme is not under a root"); + const char* sRest = NULL; + DasNode* pNear = DasCat_nearest(pCat, &sRest); + CHECK(pNear == pUiowa, "nearest is the deepest catalog that loaded"); + CHECK(streq(sRest, "nope/deeper"), "rest below it: %s", sRest ? sRest : "(null)"); + strcpy(sExpect, "uiowa"); + CHECK(streq(DasCat_lastUri(pCat), sExpect), "last URI now names the nearest node"); + + CHECK(quietResolve(pCat, "uiowa/broken") == NULL, "listed child with a missing file fails"); + pNear = DasCat_nearest(pCat, &sRest); + CHECK((pNear == pUiowa) && streq(sRest, "broken"), "nearest for a missing file"); + + CHECK(quietResolve(pCat, "uiowa/odd") == NULL, "unknown node type fails to load"); + + CHECK(quietResolve(pCat, "nope") == NULL, "bad child of the root"); + pNear = DasCat_nearest(pCat, &sRest); + CHECK((pNear == pRoot) && streq(sRest, "nope"), "nearest for a root child is the root itself"); + CHECK(streq(DasCat_lastUri(pCat), ""), "last URI is empty after a root is nearest"); + + /* A good resolve leaves nothing for nearest to find */ + CHECK(DasCat_resolve(pCat, "uiowa/geomag") != NULL, "source set resolves"); + CHECK(DasCat_nearest(pCat, &sRest) == NULL, "nearest after success is nothing"); + + /* A location given as the path is a root */ + DasNode* pDirect = DasCat_resolve(pCat, UIOWA); + CHECK(pDirect && DasNode_isCatalog(pDirect), "node by file path"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA), "the location stands in for a name: %s", DasCat_lastUri(pCat)); + CHECK(pDirect && (DasNode_pathUri(pDirect) == NULL), "a node loaded by location has no URI"); + CHECK(DasCat_nearest(pCat, &sRest) == NULL, "nearest means nothing for a location"); + CHECK(DasCat_resolve(pCat, "./" UIOWA) != pDirect, "a different spelling is a different load"); + + del_DasCat(pCat); +} + +/* ************************************************************************* */ +static void test_prefixedSeed(void) +{ + printf("TEST: a seed with a prefix\n"); + DasCat* pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setSeed(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "seed with a prefix"); + + DasNode* pTop = DasCat_resolve(pCat, NULL); + CHECK(pTop && DasNode_isCatalog(pTop), "no path: the named node"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI), "its URI"); + CHECK(pTop && streq(DasNode_pathUri(pTop), UIOWA_URI), "node knows its given URI"); + + DasNode* pSrc = DasCat_resolve(pCat, "geomag/flex"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "relative to the root"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag/flex"), "relative path gets the root URI: %s", + DasCat_lastUri(pCat)); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), UIOWA_URI "/geomag/flex"), "node path URI"); + + CHECK(DasCat_resolve(pCat, UIOWA_URI "/geomag") != NULL, "full URI under the root"); + CHECK(DasCat_resolve(pCat, "site:/uiowa/geomag") != NULL, "shorthand URI under the root"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag"), "shorthand expanded"); + + CHECK(quietResolve(pCat, "tag:das2.org,2012:site:/swri/x") == NULL, "outside the root fails"); + CHECK(!DasCat_covers(pCat, DasCat_lastUri(pCat)), "and the policy says so"); + CHECK(DasCat_covers(pCat, UIOWA_URI "/anything"), "under the root is covered"); + + CHECK(quietResolve(pCat, "geomag/nope") == NULL, "bad child under the root"); + const char* sRest = NULL; + DasNode* pNear = DasCat_nearest(pCat, &sRest); + CHECK(pNear && DasNode_isCatalog(pNear) && streq(sRest, "nope"), "nearest under a named root: %s", + sRest ? sRest : "(null)"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag"), "nearest URI under a named root"); + del_DasCat(pCat); + + /* The site file as the site node: the bare shorthand names the node, + the node's separator builds the children, so names come out as + the live catalog spells them */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setSeed(pCat, SITE, "site:") == DAS_OKAY, "seed named by bare shorthand"); + CHECK(streq(DasCat_prefix(pCat), D2URI_SITES), "bare shorthand is the node name: %s", DasCat_prefix(pCat)); + pSrc = DasCat_resolve(pCat, "uiowa/geomag/flex"); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), UIOWA_URI "/geomag/flex"), "children spelled as the live catalog: %s", + pSrc ? DasNode_pathUri(pSrc) : "(null)"); + CHECK(DasCat_resolve(pCat, "site:/uiowa/geomag/flex") == pSrc, "full URI reaches the same node"); + CHECK(DasCat_resolve(pCat, "site:uiowa/geomag/flex") == pSrc, "slash left out, same node"); + CHECK(DasCat_resolve(pCat, "site:") == DasCat_resolve(pCat, NULL), "bare shorthand resolves the root"); + del_DasCat(pCat); +} + +/* ************************************************************************* */ +static void test_mountUnderSeed(void) +{ + printf("TEST: mounts in a seed's name space\n"); + const char* sGeomag = "test/das_fetch/geomag_flex_0.7.json"; + + /* A root seed: names are paths from it, and a mount at one of them is + consulted before the listing, however the path is spelled */ + DasCat* pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setSeed(pCat, SITE, NULL) == DAS_OKAY, "root seed"); + CHECK(DasCat_mount(pCat, sGeomag, "uiowa/broken") == DAS_OKAY, "mount at a path from the root"); + DasNode* pNode = DasCat_resolve(pCat, "uiowa/broken"); + CHECK(pNode && DasNode_isStreamSrc(pNode), "the mount shadows a listed child whose file is missing"); + CHECK(streq(DasCat_lastUri(pCat), "uiowa/broken"), "named by the path: %s", DasCat_lastUri(pCat)); + CHECK(pNode && streq(DasNode_pathUri(pNode), "uiowa/broken"), "the node carries the mount name"); + CHECK(DasCat_resolve(pCat, "uiowa/broken") == pNode, "and is kept"); + DasNode* pFlex = DasCat_resolve(pCat, "uiowa/geomag/flex"); + CHECK(pFlex && DasNode_isStreamSrc(pFlex) && (pFlex != pNode), "paths off the mount still walk the seed"); + CHECK(DasCat_covers(pCat, "uiowa/broken/deeper"), "under the mount is covered"); + del_DasCat(pCat); + + /* A prefixed seed: the relative spelling and the full URI reach the + same mount */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setSeed(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "prefixed seed"); + CHECK(DasCat_mount(pCat, sGeomag, "site:/uiowa/broken") == DAS_OKAY, "mount under the prefix"); + pNode = DasCat_resolve(pCat, "broken"); + CHECK(pNode && DasNode_isStreamSrc(pNode), "relative path through the mount"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/broken"), "composed name: %s", DasCat_lastUri(pCat)); + CHECK(DasCat_resolve(pCat, UIOWA_URI "/broken") == pNode, "full URI is the same node"); + CHECK(DasCat_resolve(pCat, "site:/uiowa/broken") == pNode, "shorthand too"); + del_DasCat(pCat); +} + +/* ************************************************************************* */ +static void test_mounts(void) +{ + printf("TEST: mounts on the federated catalog (mounted side only, offline)\n"); + DasCat* pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_mount(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "mount"); + + /* A mount does not move where relative paths start; the prefix does, + and a path from there lands on the mount */ + CHECK(DasCat_setPrefix(pCat, "site:/uiowa/") == DAS_OKAY, "prefix at the mount, with its separator"); + DasNode* pSrc = DasCat_resolve(pCat, "geomag/flex"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "relative to the prefix, through the mount"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag/flex"), "URI under the mount"); + + /* Full URIs under the mount never touch the network */ + CHECK(DasCat_resolve(pCat, "site:/uiowa/geomag") != NULL, "full URI through the mount"); + CHECK(DasCat_covers(pCat, "tag:das2.org,2012:site:/swri"), "a mount leaves the rest of the catalog reachable"); + del_DasCat(pCat); + + /* With a prefix relative paths start there, and land on the mount */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_setPrefix(pCat, "site:/") == DAS_OKAY, "prefix"); + CHECK(DasCat_mount(pCat, UIOWA, "site:/uiowa") == DAS_OKAY, "mount after prefix"); + pSrc = DasCat_resolve(pCat, "uiowa/geomag/flex"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "prefix + path through the mount"); + CHECK(streq(DasCat_lastUri(pCat), UIOWA_URI "/geomag/flex"), "URI: %s", DasCat_lastUri(pCat)); + del_DasCat(pCat); + + /* Two mounts, the longer URI must win when one is under the other */ + pCat = new_DasCat(NULL, "TestCat"); + CHECK(DasCat_mount(pCat, SITE, "site:/uiowa") == DAS_OKAY, "wide mount of the wrong file"); + CHECK(DasCat_mount(pCat, "test/das_fetch/cat/geomag.json", "site:/uiowa/geomag") == DAS_OKAY, + "narrow mount"); + DasNode* pNode = DasCat_resolve(pCat, "site:/uiowa/geomag/flex"); + CHECK(pNode && DasNode_isStreamSrc(pNode), "narrow mount shadows the wide one"); + del_DasCat(pCat); +} + +/* ************************************************************************* */ +/* The user directory and the cache default built on it */ + +static bool endsWith(const char* s, const char* sTail) +{ + size_t uS = strlen(s), uT = strlen(sTail); + return (uS >= uT) && (strcmp(s + uS - uT, sTail) == 0); +} + +static void test_userdir(void) +{ + printf("TEST: user directory and the session cache default\n"); + + /* The default, unless this test's environment set DAS_USERDIR */ + const char* sEnv = getenv("DAS_USERDIR"); + if((sEnv == NULL)||(sEnv[0] == '\0')) + CHECK(endsWith(das_userdir(), DAS_DSEPS ".das"), "default is .das under home, got %s", das_userdir()); + else + CHECK(streq(das_userdir(), sEnv), "DAS_USERDIR is honored, got %s", das_userdir()); + + das_set_userdir(TEST_USERDIR); + CHECK(streq(das_userdir(), TEST_USERDIR), "setter is honored, got %s", das_userdir()); + + DasCat* pCat = new_DasCat(NULL, NULL); + CHECK(streq(DasCat_cacheDir(pCat), TEST_USERDIR DAS_DSEPS "cache" DAS_DSEPS "nodes"), + "new session caches under the user directory, got %s", + DasCat_cacheDir(pCat) ? DasCat_cacheDir(pCat) : "(off)"); + CHECK(DasCat_cacheAge(pCat) == DASCAT_DEF_CACHE_SEC, "default cache age"); + + DasCat_setCache(pCat, DasCat_cacheDir(pCat), 0); /* aliasing is allowed */ + CHECK(streq(DasCat_cacheDir(pCat), TEST_USERDIR DAS_DSEPS "cache" DAS_DSEPS "nodes"), + "age change keeps the directory"); + CHECK(DasCat_cacheAge(pCat) == 0, "age changed to always fetch"); + + DasCat_setCache(pCat, "elsewhere", 5); + CHECK(streq(DasCat_cacheDir(pCat), "elsewhere"), "directory moved"); + CHECK(DasCat_cacheAge(pCat) == 5, "age set with the move"); + + DasCat_setCache(pCat, NULL, 0); + CHECK(DasCat_cacheDir(pCat) == NULL, "NULL turns the cache off"); + DasCat_setCache(pCat, "", 0); + CHECK(DasCat_cacheDir(pCat) == NULL, "empty string turns the cache off"); + del_DasCat(pCat); + + /* A later change does not move a session already made */ + pCat = new_DasCat(NULL, NULL); + das_set_userdir("moved_after_the_fact"); + CHECK(streq(DasCat_cacheDir(pCat), TEST_USERDIR DAS_DSEPS "cache" DAS_DSEPS "nodes"), + "session keeps the directory it was made with"); + del_DasCat(pCat); + + das_set_userdir(NULL); + CHECK(!streq(das_userdir(), "moved_after_the_fact"), "NULL restores the default"); +} + +/* ************************************************************************* */ +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + + test_userdir(); + test_isUrlOrFile(); + test_policy(); + test_seed(); + test_prefixedSeed(); + test_mountUnderSeed(); + test_mounts(); + + printf("%d checks passed, %d failed\n", g_nPass, g_nFail); + return (g_nFail == 0) ? 0 : 1; +} diff --git a/test/TestCatalog.c b/test/TestCatalog.c index 1fb057f..d71e265 100644 --- a/test/TestCatalog.c +++ b/test/TestCatalog.c @@ -34,6 +34,7 @@ #include #include #include +#include /* An example fully qualified das federated catalog note fragment: @@ -70,94 +71,80 @@ int main(int argc, char** argv) { /* Exit on errors, log info messages and above */ das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_INFO, NULL); + + /* Children are loaded by a session, which holds the agent string and + the node cache; the nodes themselves no longer carry either */ + DasCat* pCat = new_DasCat(NULL, sAgent); /* Simple catalog loads, direct URLs */ sUrl = "https://das2.org/catalog/das/site/uiowa/cassini/ephemeris/saturn.json"; - /* In the global catalog this node is name: + /* In the global catalog this node is named: * * tag:das2.org,2012:site:/uiowa/ephemeris/saturn - * - * But were are loading it is if it was an isolated item so we can name it - * whatever we want, hence doggy + * + * But we are loading it by location, as an isolated item, so it is a + * root of its own with no URI. The session owns + * every node it loads; del_DasCat() at the end frees them all. */ - DasNode* pCasEphem = new_RootNode_url(sUrl, "doggy", NULL, sAgent); + DasNode* pCasEphem = DasCat_resolve(pCat, sUrl); nTest += info_or_exit(sUrl, pCasEphem, "Cassini Saturn Ephemeris", nTest); - printf("TEST %d: Deleting a single node", nTest); - del_RootNode(pCasEphem); pCasEphem = NULL; - printf(" [OKAY]\n"); - nTest += 1; - sUrl = "https://das2.org/catalog/das/site/uiowa/juno/wav.json"; - DasNode* pWavCat = new_RootNode_url(sUrl, "whattie", NULL, sAgent); + DasNode* pWavCat = DasCat_resolve(pCat, sUrl); nTest += info_or_exit(sUrl, pWavCat, "Juno Waves Catalog", nTest); - const DasNode* pSurvey = DasNode_subNode(pWavCat, "survey", NULL, sAgent); + const DasNode* pSurvey = DasCat_subNode(pCat, pWavCat, "survey"); nTest += info_or_exit(sUrl, pSurvey, "Juno Waves Survey Data Source", nTest); printf("TEST %d: (Not) Deleting a non-root ", nTest); - del_RootNode((DasNode*)pSurvey); /* Erasing const for testing purposes */ + del_DasNode((DasNode*)pSurvey); /* Erasing const for testing purposes */ /* Shouldn't be done in app code */ - if(pSurvey->sURL[0] != '\0') printf(" ...looks like it's still here [OKAY]\n"); + if(DasNode_srcUrl(pSurvey)[0] != '\0') printf(" ...looks like it's still here [OKAY]\n"); else printf("... [FAILED]"); nTest += 1; - printf("TEST %d: Deleting a catalog node", nTest); - del_RootNode(pWavCat); pWavCat = NULL; pSurvey = NULL; - printf(" [OKAY]\n"); - nTest += 1; - - - /* Central root catalog*/ - DasNode* pRoot = new_RootNode(sUri, NULL, sAgent); - /* Can't call info or exit because root URI is null */ + /* The compiled in seed, the root of the das catalog */ + DasNode* pRoot = DasCat_seed(pCat); + /* Can't call info or exit because the root has no URI */ if(pRoot == NULL){ printf("TEST %d: Find and load root node [FAILED]\n", nTest); return nTest; } else{ - printf("TEST %d: Root node loaded from %s [OKAY]\n", nTest, pRoot->sURL); + printf("TEST %d: Root node loaded from %s [OKAY]\n", nTest, DasNode_srcUrl(pRoot)); } nTest += 1; /* Extra cassini check */ sUrl = "tag:das2.org,2012:site:/uiowa/cassini/rpws/survey"; - const DasNode* pCasSur = DasNode_subNode(pRoot, sUrl, NULL, sAgent); + const DasNode* pCasSur = DasCat_subNode(pCat, pRoot, sUrl); nTest += info_or_exit(sUri, pCasSur, "Cassini/survey", nTest); - /*das2 root */ + /*das root */ sUri = "tag:das2.org,2012:"; - const DasNode* pDas2 = DasNode_subNode(pRoot, sUri, NULL, sAgent); + const DasNode* pDas2 = DasCat_subNode(pCat, pRoot, sUri); nTest += info_or_exit(sUri, pDas2, "Root", nTest); /* spase root */ sUri = "tag:spase-group.org,2018:spase://"; - const DasNode* pSpase = DasNode_subNode(pRoot, sUri, NULL, sAgent); + const DasNode* pSpase = DasCat_subNode(pCat, pRoot, sUri); nTest += info_or_exit(sUri, pSpase, "Root", nTest); /* cdaweb root */ sUri = "tag:cdaweb.sci.gsfc.nasa.gov,2018:"; - const DasNode* pCdaWeb = DasNode_subNode(pRoot, sUri, NULL, sAgent); + const DasNode* pCdaWeb = DasCat_subNode(pCat, pRoot, sUri); nTest += info_or_exit(sUri, pCdaWeb, "Root", nTest); /* Deep dig from existing root */ sUri = "tag:das2.org,2012:site:/uiowa/juno/wav/uncalibrated/hrs"; - const DasNode* pWavUcalHrs = DasNode_subNode(pRoot, sUri, NULL, sAgent); + const DasNode* pWavUcalHrs = DasCat_subNode(pCat, pRoot, sUri); nTest += info_or_exit(sUri, pWavUcalHrs, "Juno Waves Uncalibrated HFWBR", nTest); - /* deleting top root */ - printf("TEST %d: Deleting URI based top root ", nTest); - del_RootNode(pRoot); - pRoot = NULL; pDas2 = pSpase = pCdaWeb = pWavUcalHrs = NULL; - printf(" [OKAY]\n"); - nTest += 1; - - - /* direct deep dig to ferret out memory leaks */ + /* A deep path through the session: the levels walked above stay loaded */ sUri = "tag:das2.org,2012:site:/uiowa/juno/wav/survey/das2"; - DasNode* pSurvey2 = new_RootNode(sUri, NULL, sAgent); + DasNode* pSurvey2 = DasCat_resolve(pCat, sUri); nTest += info_or_exit(sUri, pSurvey2, "Waves Survey Deep Read", nTest); /* Get a Query definition */ @@ -172,12 +159,8 @@ int main(int argc, char** argv) { } nTest += 1; - /* And remove it */ - printf("TEST %d: Deleting deep URI direct lookup", nTest); - del_RootNode(pSurvey2); - printf(" [OKAY]\n"); - /* TODO: Add more JSON catalog tests */ + del_DasCat(pCat); return 0; } diff --git a/test/TestHttp.c b/test/TestHttp.c new file mode 100644 index 0000000..0d5d9fb --- /dev/null +++ b/test/TestHttp.c @@ -0,0 +1,202 @@ +/** @file TestHttp.c Unit tests for http.h that need no network: the URL + * parser behind DasHttpResp_init() and the response header lookup. */ + +/* Author: Chris Piker , via Claude Fable 5.1 + * + * This file is intended to demonstrate an interface. This is free + * and unencumbered software released into the public domain + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this file, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this file dedicate any and all copyright interest in this file to + * the public domain. We make this dedication for the benefit of the + * public at large and to the detriment of our heirs and successors. We + * intend this dedication to be an overt act of relinquishment in + * perpetuity of all present and future rights to this file under + * copyright law. + * + * THIS FILE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * + * For more information, please refer to + */ + +/* The parser has fixed size fields. The property under test is that a part + * which fits is kept whole and a part which does not is refused with an + * error message, never silently shortened. A shortened query would reach + * the server as a different request and a shortened dataset would match + * the wrong saved password. */ + +#include +#include +#include +#include + +#include +#include + +static int g_nFail = 0; +static int g_nPass = 0; + +#define CHECK(cond, ...) do{ \ + if(cond){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } \ +}while(0) + +static bool streq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return (a == b); + return strcmp(a, b) == 0; +} + +/* Parse into a cleared response. Any error string is freed before the + * next call so a leak checker stays quiet. */ +static bool parse(DasHttpResp* pRes, const char* sUrl) +{ + if(pRes->sError){ free(pRes->sError); pRes->sError = NULL; } + memset(pRes, 0, sizeof(DasHttpResp)); + return DasHttpResp_init(pRes, sUrl); +} + +/* A URL whose named part is exactly uLen characters long */ +static void mkUrl(char* sBuf, size_t uBuf, const char* sPart, size_t uLen) +{ + size_t u = 0; + if(strcmp(sPart, "scheme") == 0){ + for(; u < uLen; ++u) sBuf[u] = 'h'; + snprintf(sBuf + u, uBuf - u, "://host/p"); + memcpy(sBuf, "http", 4); /* still not a valid scheme, length is the point */ + return; + } + if(strcmp(sPart, "host") == 0){ + u = snprintf(sBuf, uBuf, "http://"); + for(size_t i = 0; i < uLen; ++i) sBuf[u++] = 'h'; + snprintf(sBuf + u, uBuf - u, "/p"); + return; + } + if(strcmp(sPart, "path") == 0){ + u = snprintf(sBuf, uBuf, "http://host/"); + for(size_t i = 1; i < uLen; ++i) sBuf[u++] = 'p'; + snprintf(sBuf + u, uBuf - u, "?q=1"); + return; + } + if(strcmp(sPart, "query") == 0){ + u = snprintf(sBuf, uBuf, "http://host/p?"); + for(size_t i = 0; i < uLen; ++i) sBuf[u++] = 'q'; + sBuf[u] = '\0'; + return; + } + if(strcmp(sPart, "dataset") == 0){ + u = snprintf(sBuf, uBuf, "http://host/p?server=dataset&dataset="); + for(size_t i = 0; i < uLen; ++i) sBuf[u++] = 'd'; + snprintf(sBuf + u, uBuf - u, "&start_time=2026"); + return; + } +} + +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + + DasHttpResp res; + memset(&res, 0, sizeof(DasHttpResp)); + char sUrl[DASURL_SZ_QUERY + 256]; + char sOut[DASURL_SZ_QUERY + 256]; + + printf("INFO: Parsing well formed URLs\n"); + + CHECK(parse(&res, "http://host/path?a=1"), "plain http parses"); + CHECK(streq(res.url.sScheme, "http"), "scheme"); + CHECK(streq(res.url.sHost, "host"), "host"); + CHECK(streq(res.url.sPort, "80"), "default http port, got %s", res.url.sPort); + CHECK(streq(res.url.sPath, "/path"), "path, got %s", res.url.sPath); + CHECK(streq(res.url.sQuery, "a=1"), "query, got %s", res.url.sQuery); + CHECK(res.url.sDataset[0] == '\0', "no dataset"); + CHECK(streq(das_url_toStr(&res.url, sOut, sizeof(sOut)), "http://host/path?a=1"), + "round trip drops the default port, got %s", sOut); + + CHECK(parse(&res, "https://space.physics.uiowa.edu:8443/das/stream/source/tracers/flex" + "?read.time.min=2026-01-01T00%3A00"), "https with port parses"); + CHECK(streq(res.url.sPort, "8443"), "explicit port, got %s", res.url.sPort); + CHECK(streq(res.url.sPath, "/das/stream/source/tracers/flex"), "path, got %s", res.url.sPath); + CHECK(streq(res.url.sQuery, "read.time.min=2026-01-01T00%3A00"), "query, got %s", res.url.sQuery); + CHECK(streq(das_url_toStr(&res.url, sOut, sizeof(sOut)), + "https://space.physics.uiowa.edu:8443/das/stream/source/tracers/flex?read.time.min=2026-01-01T00%3A00"), + "round trip keeps an explicit port, got %s", sOut); + + CHECK(parse(&res, "https://host"), "bare host parses"); + CHECK(streq(res.url.sPort, "443"), "default https port, got %s", res.url.sPort); + CHECK(res.url.sPath[0] == '\0', "empty path"); + CHECK(res.url.sQuery[0] == '\0', "empty query"); + + CHECK(parse(&res, "http://h/x?server=dataset&dataset=Juno/WAV/Survey&start_time=1"), + "das2.2 style query parses"); + CHECK(streq(res.url.sDataset, "Juno/WAV/Survey"), "dataset stops at &, got %s", res.url.sDataset); + + printf("INFO: Refusing malformed URLs\n"); + + CHECK(!parse(&res, "ftp://host/x") && res.sError, "ftp scheme refused"); + CHECK(!parse(&res, "http://") && res.sError, "empty host refused"); + CHECK(!parse(&res, "http://host:99999/x") && res.sError, "port out of range refused"); + CHECK(!parse(&res, "http://host:abc/x") && res.sError, "non numeric port refused"); + CHECK(!parse(&res, "http://host:/x") && res.sError, "empty port refused"); + + printf("INFO: Parts at the size limit fit, one byte over is refused\n"); + + struct { const char* sPart; size_t uMax; } aLim[] = { + {"host", DASURL_SZ_HOST}, {"path", DASURL_SZ_PATH}, + {"query", DASURL_SZ_QUERY}, {"dataset", DASURL_SZ_DATASET} + }; + for(size_t i = 0; i < sizeof(aLim)/sizeof(aLim[0]); ++i){ + mkUrl(sUrl, sizeof(sUrl), aLim[i].sPart, aLim[i].uMax); + CHECK(parse(&res, sUrl), "%s of %zu bytes parses: %s", aLim[i].sPart, aLim[i].uMax, + res.sError ? res.sError : ""); + + mkUrl(sUrl, sizeof(sUrl), aLim[i].sPart, aLim[i].uMax + 1); + bool bOk = parse(&res, sUrl); + CHECK(!bOk, "%s of %zu bytes is refused", aLim[i].sPart, aLim[i].uMax + 1); + CHECK(!bOk && res.sError && strstr(res.sError, aLim[i].sPart), + "refusal names the %s, got: %s", aLim[i].sPart, res.sError ? res.sError : "(null)"); + } + + /* The scheme is checked by value before anything else, so an over long + one is refused either way; make sure it is refused and not overrun */ + mkUrl(sUrl, sizeof(sUrl), "scheme", DASURL_SZ_SCHEME + 1); + CHECK(!parse(&res, sUrl) && res.sError, "over long scheme refused"); + if(res.sError){ free(res.sError); res.sError = NULL; } + + printf("INFO: Header lookup on a saved response\n"); + + /* The lookup only reads sHeaders, so a response can be staged by hand. + Mixed case and odd spacing on purpose, both occur in the wild. */ + memset(&res, 0, sizeof(DasHttpResp)); + res.sHeaders = strdup( + "HTTP/1.1 200 OK\r\n" + "Server: Apache\r\n" + "etag: \"5a6-607e10c29b347\"\r\n" + "Last-Modified: Tue, 17 Oct 2023 03:23:33 GMT \r\n" + "Content-Type: application/json\r\n" + "\r\n" + ); + CHECK(DasHttpResp_header(&res, "ETag", sOut, sizeof(sOut)) && streq(sOut, "\"5a6-607e10c29b347\""), + "ETag found without regard to case, got '%s'", sOut); + CHECK(DasHttpResp_header(&res, "last-modified", sOut, sizeof(sOut)) && + streq(sOut, "Tue, 17 Oct 2023 03:23:33 GMT"), "Last-Modified found and trimmed, got '%s'", sOut); + CHECK(DasHttpResp_header(&res, "Content-Type", sOut, sizeof(sOut)) && streq(sOut, "application/json"), + "Content-Type found"); + CHECK(!DasHttpResp_header(&res, "Content-Length", sOut, sizeof(sOut)) && (sOut[0] == '\0'), + "absent header reports false with an empty buffer"); + CHECK(!DasHttpResp_header(&res, "Content", sOut, sizeof(sOut)), "a prefix of a name is not a match"); + CHECK(!DasHttpResp_header(&res, "ETag", sOut, 8), "a value that does not fit is refused, not cut"); + free(res.sHeaders); + res.sHeaders = NULL; + CHECK(!DasHttpResp_header(&res, "ETag", sOut, sizeof(sOut)), "no headers, no match"); + + printf("INFO: %d checks passed, %d failed\n", g_nPass, g_nFail); + return (g_nFail == 0) ? 0 : 13; +} diff --git a/test/TestSource.c b/test/TestSource.c new file mode 100644 index 0000000..90328da --- /dev/null +++ b/test/TestSource.c @@ -0,0 +1,862 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das2 C Library. + * + * Das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * Das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* Sources, requisitions and URLs from the three saved source nodes under + * test/das_fetch, entirely offline. Run from the repository top. + * + * geomag is the current layout with examples and mirrors, hsk the current + * layout at scale (73 variables), juno_survey the 2018 layout with long + * property names, flag packed options and protocol level examples. The + * URL cases are the ones utilities/das_fetch.c has always been held to, + * now asked of the library directly. */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include + +#include +#include +#include +#include +#include "../das3/src_priv.h" + +#define GEOMAG "test/das_fetch/geomag_flex_0.7.json" +#define HSK "test/das_fetch/hsk_flex_0.7.json" +#define JUNO "test/das_fetch/juno_survey_0.6.json" + +#define GB "http://127.0.0.1:8765/data/geomag.d2s" +#define HB "https://space.physics.uiowa.edu/das/stream/source/tracers/tracers-2/l1a/cdpu/hsk/flex" +#define JB "http://127.0.0.1:8765/data/juno_survey.d2s?server=dataset&dataset=Juno/WAV/Survey" + +static int g_nFail = 0; +static int g_nPass = 0; + +#define CHECK(cond, ...) do{ \ + if(cond){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL line %d: ", __LINE__); printf(__VA_ARGS__); printf("\n"); } \ +}while(0) + +static bool streq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return (a == b); + return strcmp(a, b) == 0; +} + +static char* readFile(const char* sFile, size_t* puLen) +{ + FILE* pIn = fopen(sFile, "rb"); + if(pIn == NULL){ printf(" Can't open %s, run from the repo top\n", sFile); return NULL; } + fseek(pIn, 0, SEEK_END); + long nLen = ftell(pIn); + fseek(pIn, 0, SEEK_SET); + char* sBuf = (char*)malloc(nLen + 1); + *puLen = fread(sBuf, 1, nLen, pIn); + fclose(pIn); + sBuf[*puLen] = '\0'; + return sBuf; +} + +static DasSrc* loadSrc(const char* sFile, const char* sPath) +{ + size_t uLen = 0; + char* sDoc = readFile(sFile, &uLen); + if(sDoc == NULL) return NULL; + DasSrc* pSrc = new_DasSrc_json(sDoc, uLen, sPath, sFile); + free(sDoc); + return pSrc; +} + +static const DasSrcProp* byKey(const DasSrc* pSrc, const char* sKey) +{ + for(size_t u = 0; u < DasSrc_propCount(pSrc); ++u) + if(streq(DasSrcProp_key(DasSrc_prop(pSrc, u)), sKey)) return DasSrc_prop(pSrc, u); + return NULL; +} + +static const DasSrcCtrl* ctrlByKey(const DasSrc* pSrc, const char* sKey) +{ + const DasSrcProp* p = byKey(pSrc, sKey); + return p ? DasSrcProp_ctrl(p) : NULL; +} + +static size_t ctrlCount(const DasSrc* pSrc, int nSec) +{ + size_t n = 0; + for(size_t u = 0; u < DasSrc_propCount(pSrc); ++u){ + const DasSrcProp* p = DasSrc_prop(pSrc, u); + if((nSec >= 0) && (DasSrcProp_section(p) != (das_src_sec_e)nSec)) continue; + if(DasSrcProp_ctrl(p) != NULL) ++n; + } + return n; +} + +static DasErrCode chk(const DasSrcCtrl* pC, const char* sVal) +{ + return DasSrcCtrl_check(pC, sVal, NULL, 0); +} + +/* A KEY=VAL list to a URL, the das_fetch_test.sh shape. nExample seeds the + requisition, -1 for none. Errors are expected in some cases, so the log + is quiet for the duration. */ +static bool urlFor( + const DasSrc* pSrc, int nExample, const char** aKeyVal, const char** aRaw, + char* sUrl, size_t uLen +){ + DasReq* pReq = (nExample >= 0) ? DasSrc_newReq(pSrc, nExample) : new_DasReq(); + if(pReq == NULL) return false; + for(size_t u = 0; aKeyVal && aKeyVal[u]; ++u){ + const char* pEq = strchr(aKeyVal[u], '='); + char sKey[128]; + if(pEq){ + size_t n = (size_t)(pEq - aKeyVal[u]); + memcpy(sKey, aKeyVal[u], n); sKey[n] = '\0'; + DasReq_set(pReq, sKey, pEq + 1); + } + else{ + DasReq_set(pReq, aKeyVal[u], "true"); + } + } + for(size_t u = 0; aRaw && aRaw[u]; u += 2) DasReq_setRaw(pReq, aRaw[u], aRaw[u+1]); + daslog_setlevel(DASLOG_CRIT); + size_t n = DasStreamSrc_url(pSrc, pReq, 0, sUrl, uLen); + daslog_setlevel(DASLOG_WARN); + del_DasReq(pReq); + return n > 0; +} + +#define URL_IS(src, ex, keys, raw, want) do{ \ + char _s[2048]; const char* _k[] = keys; const char* _r[] = raw; \ + bool _ok = urlFor(src, ex, _k, _r, _s, sizeof(_s)); \ + CHECK(_ok && streq(_s, want), "\n want %s\n got %s", want, _ok ? _s : "(no url)"); \ +}while(0) + +#define URL_FAILS(src, ex, keys) do{ \ + char _s[2048]; const char* _k[] = keys; \ + CHECK(!urlFor(src, ex, _k, NULL, _s, sizeof(_s)), "expected failure, got %s", _s); \ +}while(0) + +#define K(...) {__VA_ARGS__, NULL} +#define NONE {NULL} + +/* ************************************************************************* */ + +static void test_grammar(void) +{ + printf("Key grammar:\n"); + das_src_sec_e sec; + CHECK(_das_src_secByName("coord", &sec) && (sec == das_src_sec_coord), "coord"); + CHECK(_das_src_secByName("Coordinates", &sec) && (sec == das_src_sec_coord), "Coordinates"); + CHECK(_das_src_secByName("fmt", &sec) && (sec == das_src_sec_format), "fmt"); + CHECK(_das_src_secByName("opts", &sec) && (sec == das_src_sec_option), "opts"); + CHECK(!_das_src_secByName("time", &sec), "time is not a section"); + CHECK(!_das_src_secByName("", &sec), "empty is not a section"); + + CHECK(streq(das_src_sec_name(das_src_sec_coord), "coords"), "coords name"); + CHECK(streq(das_src_sec_name(das_src_sec_format), "formats"), "formats name"); + CHECK(streq(das_src_sec_name(das_src_sec_size), ""), "sentinel has no name"); + + CHECK(streq(_das_src_propName("Minimum"), "min"), "Minimum -> min"); + CHECK(streq(_das_src_propName("inter"), "inter"), "inter stays"); + CHECK(streq(_das_src_propName("enable"), "enabled"), "enable -> enabled"); + CHECK(streq(_das_src_propName("delim"), "delim"), "unknown passes through"); + CHECK(_das_src_propEq("maximum", "MAX"), "maximum == MAX"); + CHECK(!_das_src_propEq("res", "inter"), "res != inter"); + + char aParts[3][DAS_MAX_ID_BUFSZ]; + CHECK(_das_src_splitKey("coords.time.min", aParts) == 3, "three parts"); + CHECK(streq(aParts[1], "time") && streq(aParts[2], "min"), "parts content"); + CHECK(_das_src_splitKey("lshell", aParts) == 1, "one part"); + CHECK(_das_src_splitKey("a.b.c.d", aParts) == 0, "four parts is malformed"); + CHECK(_das_src_splitKey("a..c", aParts) == 0, "empty part is malformed"); + CHECK(_das_src_splitKey("", aParts) == 0, "empty key is malformed"); + CHECK(_das_src_splitKey(NULL, aParts) == 0, "NULL key is malformed"); +} + +static void test_scalar(void) +{ + printf("JSON scalars:\n"); + const char* sDoc = "{\"st\":\"x\",\"nm\":86.4,\"in\":42,\"tr\":true,\"fa\":false," + "\"nl\":null,\"ar\":[1],\"di\":{},\"x\":\"one\",\"ls\":[[7]]}"; + DasJdo* pDom = das_json_parse(sDoc, strlen(sDoc)); + CHECK(pDom != NULL, "parse"); + char sBuf[32]; + CHECK(_das_json_scalar(DasJdo_get(pDom, "x"), sBuf, 32) && streq(sBuf, "one"), "one character key"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "ls/0/0"), sBuf, 32) && streq(sBuf, "7"), "index then index"); + CHECK(DasJdo_get(pDom, "12/x") == NULL, "index past the end"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "st"), sBuf, 32) && streq(sBuf, "x"), "string"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 32) && streq(sBuf, "86.4"), "number text kept"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "in"), sBuf, 32) && streq(sBuf, "42"), "integer"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "tr"), sBuf, 32) && streq(sBuf, "true"), "true"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "fa"), sBuf, 32) && streq(sBuf, "false"), "false"); + CHECK(!_das_json_scalar(DasJdo_get(pDom, "nl"), sBuf, 32), "null is not a scalar"); + CHECK(!_das_json_scalar(DasJdo_get(pDom, "ar"), sBuf, 32), "array is not a scalar"); + CHECK(!_das_json_scalar(NULL, sBuf, 32), "NULL is not a scalar"); + CHECK(_das_json_scalar(DasJdo_get(pDom, "nm"), sBuf, 3) && streq(sBuf, "86"), "number cut to fit"); + free(pDom); +} + +static void test_helpers(void) +{ + printf("URL encoding and time spans:\n"); + char sBuf[64]; + CHECK(das_url_encode("2026-01-02T14:30", sBuf, sizeof(sBuf)) == 18, "encoded length"); + CHECK(streq(sBuf, "2026-01-02T14%3A30"), "colon encoded, got %s", sBuf); + CHECK(das_url_encode("a b&c", sBuf, sizeof(sBuf)) == 9, "space and ampersand"); + CHECK(streq(sBuf, "a%20b%26c"), "got %s", sBuf); + CHECK(das_url_encode("abc", sBuf, 2) == 3, "needed length when short"); + CHECK(streq(sBuf, "a"), "cut short and terminated, got '%s'", sBuf); + CHECK(das_url_encode("abc", NULL, 0) == 3, "measure only"); + CHECK(das_url_encode("", sBuf, sizeof(sBuf)) == 0 && streq(sBuf, ""), "empty"); + + das_time dtB, dtE; + char sB[40], sE[40]; +#define SPAN_IS(str, b, e) do{ \ + bool _ok = dt_parsespan(str, &dtB, &dtE); \ + if(_ok){ dt_isoc(&dtB, sB, 40, 0); dt_isoc(&dtE, sE, 40, 0); } \ + CHECK(_ok && streq(sB, b) && streq(sE, e), "%s -> %s %s, want %s %s", str, \ + _ok ? sB : "(refused)", _ok ? sE : "", b, e); \ +}while(0) + SPAN_IS("2026", "2026-01-01T00:00:00", "2027-01-01T00:00:00"); + SPAN_IS("2026-02", "2026-02-01T00:00:00", "2026-03-01T00:00:00"); + SPAN_IS("2026-01-02", "2026-01-02T00:00:00", "2026-01-03T00:00:00"); + SPAN_IS("2026-12-31", "2026-12-31T00:00:00", "2027-01-01T00:00:00"); + SPAN_IS("2026-01-02T14", "2026-01-02T14:00:00", "2026-01-02T15:00:00"); + SPAN_IS("2026-01-02T23", "2026-01-02T23:00:00", "2026-01-03T00:00:00"); + SPAN_IS("2026-01-02T14:30", "2026-01-02T14:30:00", "2026-01-02T14:31:00"); + SPAN_IS("2026-01-02T14:30:15Z", "2026-01-02T14:30:15", "2026-01-02T14:30:16"); + SPAN_IS("2026-002", "2026-01-02T00:00:00", "2026-01-03T00:00:00"); + SPAN_IS("2026-002T06", "2026-01-02T06:00:00", "2026-01-02T07:00:00"); + CHECK(!dt_parsespan("2026-01-02T14:30:15.5", NULL, NULL), "fractions have no span"); + CHECK(!dt_parsespan("T14", NULL, NULL), "a time needs a date"); + CHECK(!dt_parsespan("now", NULL, NULL), "now is not a span"); + CHECK(!dt_parsespan("", NULL, NULL), "empty"); + CHECK(!dt_parsespan("2026-01-02,2026-01-03", NULL, NULL), "a range is not a span"); + CHECK(dt_parsespan("2026-01-02", NULL, NULL), "outputs are optional"); +} + +static void test_geomag(void) +{ + printf("geomag, current layout:\n"); + DasSrc* pSrc = loadSrc(GEOMAG, "tag:das2.org,2012:site:/uiowa/tracers/tracers-2/spice/geomagcoords/flex"); + CHECK(pSrc != NULL, "load"); + if(pSrc == NULL) return; + + CHECK(DasSrc_kind(pSrc) == das_src_kind_stream, "stream source"); + CHECK(streq(DasSrc_version(pSrc), "0.7"), "version"); + CHECK(streq(DasSrc_url(pSrc), GEOMAG), "url is where it came from"); + CHECK(!DasSrc_needsAuth(pSrc), "no auth"); + CHECK(DasStreamSrc_urlCount(pSrc) == 2, "two mirrors"); + CHECK(streq(DasStreamSrc_baseUrl(pSrc, 1), "http://127.0.0.1:8765/data/geomag_alt.d2s"), "second mirror"); + CHECK(DasStreamSrc_baseUrl(pSrc, 2) == NULL, "past the mirrors"); + + /* time min/max/inter, lshell enabled, then 13 format knobs */ + CHECK(ctrlCount(pSrc, -1) == 17, "17 controls, got %zu", ctrlCount(pSrc, -1)); + CHECK(DasSrc_propCount(pSrc) > 17, "fixed props are kept too, %zu total", DasSrc_propCount(pSrc)); + + /* Groups */ + CHECK(DasSrc_groupCount(pSrc, das_src_sec_coord) == 1, "one coordinate"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_data) == 1, "one variable"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_format) == 3, "three formats, got %zu", + DasSrc_groupCount(pSrc, das_src_sec_format)); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_option) == 0, "no options"); + CHECK(DasSrc_group(pSrc, das_src_sec_option, 0) == NULL, "no option group"); + + const DasSrcGroup* pG = DasSrc_findGroup(pSrc, das_src_sec_coord, "time"); + CHECK(pG != NULL, "time group"); + if(pG){ + CHECK(streq(DasSrcGroup_label(pG), "Time"), "label"); + CHECK(DasSrcGroup_enable(pG) == NULL, "coordinates have no gate"); + CHECK(DasSrcGroup_propCount(pG) == 4, "min max units inter, got %zu", DasSrcGroup_propCount(pG)); + CHECK(DasSrcGroup_ctrlCount(pG) == 3, "three of them settable"); + const DasSrcProp* pU = DasSrcGroup_findProp(pG, "units"); + CHECK(pU && (DasSrcProp_ctrl(pU) == NULL), "units is a fixed property"); + CHECK(pU && streq(DasSrcProp_value(pU), "UTC"), "with the value UTC"); + CHECK(DasSrcGroup_findProp(pG, "minimum") == DasSrcGroup_findProp(pG, "min"), "alias in findProp"); + CHECK(DasSrcGroup_prop(pG, 4) == NULL, "past the end"); + } + + pG = DasSrc_findGroup(pSrc, das_src_sec_format, "csv"); + CHECK(pG != NULL, "csv group"); + if(pG){ + const DasSrcProp* pOn = DasSrcGroup_enable(pG); + CHECK(pOn && streq(DasSrcProp_key(pOn), "formats.csv.enabled"), "the gate"); + CHECK(streq(DasSrcGroup_xorGroup(pG), "format"), "formats are exclusive"); + CHECK(DasSrcGroup_mimeCount(pG) == 1 && streq(DasSrcGroup_mime(pG, 0), "text/csv"), "mime"); + } + pG = DasSrc_findGroup(pSrc, das_src_sec_data, "lshell"); + CHECK(pG && DasSrcGroup_enable(pG) && (DasSrcGroup_xorGroup(pG) == NULL), "variable gate, not exclusive"); + + /* Properties and controls */ + const DasSrcProp* p = byKey(pSrc, "coords.time.min"); + CHECK(p != NULL, "coords.time.min"); + if(p){ + const DasSrcCtrl* pC = DasSrcProp_ctrl(p); + CHECK(pC != NULL, "has a control"); + CHECK(DasSrcCtrl_prop(pC) == p, "control points back"); + CHECK(streq(DasSrcGroup_name(DasSrcProp_group(p)), "time"), "group"); + CHECK(streq(DasSrcProp_name(p), "min"), "name"); + CHECK(streq(DasSrcCtrl_param(pC), "read.time.min"), "param via dotted name"); + CHECK(DasSrcCtrl_required(pC), "required from the protocol parameter"); + CHECK(DasSrcProp_type(p) == das_srcprop_time, "type from the parameter"); + CHECK(streq(DasSrcProp_typeName(p), "isotime"), "type name from the parameter"); + CHECK(streq(DasSrcProp_units(p), "UTC"), "UTC for a time"); + CHECK(DasSrcProp_value(p) == NULL, "null value is no default"); + CHECK(chk(pC, "2026-01-01") == DAS_OKAY, "ISO date passes"); + CHECK(chk(pC, "2026-01-01T12:00:00") == DAS_OKAY, "ISO time passes"); + CHECK(chk(pC, "now") == DAS_OKAY, "now passes"); + char sWhy[256]; + CHECK(DasSrcCtrl_check(pC, "yesterday-ish", sWhy, sizeof(sWhy)) == DASERR_SRC, "garbage fails"); + CHECK(strstr(sWhy, "coords.time.min") != NULL, "reason names the key: %s", sWhy); + } + + const DasSrcCtrl* pC = ctrlByKey(pSrc, "coords.time.inter"); + CHECK(pC != NULL, "coords.time.inter"); + if(pC){ + CHECK(DasSrcProp_type(DasSrcCtrl_prop(pC)) == das_srcprop_real, "real from the parameter"); + CHECK(streq(DasSrcProp_units(DasSrcCtrl_prop(pC)), "s"), "units string on the property"); + CHECK(chk(pC, "60") == DAS_OKAY, "60 is real"); + CHECK(chk(pC, "6e1") == DAS_OKAY, "6e1 is real"); + CHECK(chk(pC, "sixty") == DASERR_SRC, "sixty is not"); + } + + pC = ctrlByKey(pSrc, "data.lshell.enabled"); + CHECK(pC != NULL, "data.lshell.enabled"); + if(pC){ + CHECK(DasSrcProp_type(DasSrcCtrl_prop(pC)) == das_srcprop_bool, "boolean"); + CHECK(streq(DasSrcCtrl_param(pC), "read.data"), "flag set parameter"); + CHECK(!DasSrcCtrl_required(pC), "not required"); + CHECK(chk(pC, "yes") == DAS_OKAY, "yes"); + CHECK(chk(pC, "OFF") == DAS_OKAY, "OFF"); + CHECK(chk(pC, "maybe") == DASERR_SRC, "maybe"); + } + + char sBuf[64]; + p = byKey(pSrc, "formats.csv.enabled"); + CHECK(p != NULL, "formats.csv.enabled"); + if(p){ + CHECK(streq(DasSrcCtrl_param(DasSrcProp_ctrl(p)), "format.type"), "format selector param"); + const char* sTitle = DasSrcProp_title(p, sBuf, sizeof(sBuf)); + CHECK(strncmp(sTitle, "Output as ", 10) == 0, "composed format title, got '%s'", sTitle); + } + + pC = ctrlByKey(pSrc, "formats.csv.delimiter"); + CHECK(pC != NULL, "formats.csv.delimiter"); + if(pC){ + CHECK(DasSrcProp_type(DasSrcCtrl_prop(pC)) == das_srcprop_enum, "delimiter is an enum"); + CHECK(DasSrcCtrl_enumCount(pC) == 4, "four delimiters, got %zu", DasSrcCtrl_enumCount(pC)); + CHECK(DasSrcCtrl_enum(pC, 4) == NULL, "past the end"); + CHECK(streq(DasSrcCtrl_enum(pC, 0), "Comma"), "node spelling kept"); + CHECK(chk(pC, "tab") == DAS_OKAY, "enum match is case insensitive"); + CHECK(chk(pC, "colon") == DASERR_SRC, "colon is not offered"); + } + + pC = ctrlByKey(pSrc, "formats.csv.fracSecs"); + CHECK(pC != NULL, "formats.csv.fracSecs"); + if(pC){ + const char* sLo = NULL; const char* sHi = NULL; + CHECK(DasSrcProp_type(DasSrcCtrl_prop(pC)) == das_srcprop_int, "integer from the parameter"); + CHECK(DasSrcCtrl_range(pC, &sLo, &sHi), "range from the parameter"); + CHECK(streq(sLo, "0") && streq(sHi, "9"), "0 to 9, got %s to %s", sLo, sHi); + CHECK(chk(pC, "12") == DAS_OKAY, "12 passes, ranges are not enforced"); + CHECK(chk(pC, "3.5") == DASERR_SRC, "3.5 is not an integer"); + } + + /* Lookup by typed key */ + char sWhy[256]; + CHECK(DasSrc_findProp(pSrc, "time.min", sWhy, sizeof(sWhy)) == byKey(pSrc, "coords.time.min"), "time.min"); + CHECK(DasSrc_findProp(pSrc, "coord.time.minimum", NULL, 0) == byKey(pSrc, "coords.time.min"), "long form"); + CHECK(DasSrc_findProp(pSrc, "lshell", NULL, 0) == byKey(pSrc, "data.lshell.enabled"), "group toggle"); + CHECK(DasSrc_findProp(pSrc, "csv", NULL, 0) == byKey(pSrc, "formats.csv.enabled"), "format toggle"); + CHECK(DasSrc_findProp(pSrc, "serial", NULL, 0) != NULL, "unique bare prop"); + CHECK(DasSrc_findProp(pSrc, "fracSecs", sWhy, sizeof(sWhy)) == NULL, "ambiguous bare prop"); + CHECK(strstr(sWhy, "formats.das.fracSecs") && strstr(sWhy, "formats.csv.fracSecs"), "names both: %s", sWhy); + CHECK(DasSrc_findProp(pSrc, "nope", sWhy, sizeof(sWhy)) == NULL && sWhy[0], "unknown key has a reason"); + CHECK(DasSrc_findProp(pSrc, "time.units", NULL, 0) != NULL, "fixed props are found too"); + + /* Examples and requisitions */ + CHECK(DasSrc_exampleCount(pSrc) == 1, "one example"); + CHECK(streq(DasSrc_exampleLabel(pSrc, 0), "First day of 2026"), "label"); + CHECK(DasSrc_exampleLabel(pSrc, 1) == NULL, "past the examples"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasSrc_newReq(pSrc, 9) == NULL, "no example 9"); + daslog_setlevel(DASLOG_WARN); + + DasReq* pReq = DasSrc_newReq(pSrc, 0); + CHECK(pReq != NULL, "example req"); + if(pReq){ + CHECK(streq(DasReq_label(pReq), "First day of 2026"), "req carries the label"); + CHECK(streq(DasReq_srcPath(pReq), DasSrc_path(pSrc)), "stamped with the path"); + CHECK(streq(DasReq_value(pReq, "coords.time.min"), "2026-01-01"), "example setting in key form"); + CHECK(DasReq_count(pReq) == 3, "three settings, got %zu", DasReq_count(pReq)); + size_t uBad = 9; + CHECK(DasSrc_check(pSrc, pReq, &uBad) == DAS_OKAY && uBad == 0, "example checks clean"); + CHECK(DasReq_verdict(pReq, 0) == das_req_ok, "verdict ok"); + del_DasReq(pReq); + } + + pReq = DasSrc_newReq(pSrc, DASSRC_BLANK_REQ); + CHECK(pReq != NULL, "blank req"); + if(pReq){ + CHECK(DasReq_count(pReq) == 3, "lists the three required keys, got %zu", DasReq_count(pReq)); + CHECK(streq(DasReq_value(pReq, "coords.time.inter"), ""), "with empty values"); + size_t uBad = 0; + CHECK(DasSrc_check(pSrc, pReq, &uBad) == DASERR_SRC && uBad == 3, "all three bad, got %zu", uBad); + CHECK(strstr(DasReq_why(pReq, 0), "required") != NULL, "says required: %s", DasReq_why(pReq, 0)); + CHECK(DasReq_set(pReq, "time", "2026-04-01,2026-04-01T01:30") == DAS_OKAY, "range"); + CHECK(DasSrc_check(pSrc, pReq, &uBad) == DASERR_SRC && uBad == 1, "inter still bad, got %zu", uBad); + size_t uDerived = 0; + for(size_t u = 0; u < DasReq_count(pReq); ++u) if(DasReq_derived(pReq, u)) ++uDerived; + CHECK(uDerived == 2, "min and max derived, got %zu", uDerived); + DasReq_set(pReq, "inter", "60"); + CHECK(DasSrc_check(pSrc, pReq, &uBad) == DAS_OKAY, "now clean"); + char sUrl[2048]; + CHECK(DasStreamSrc_url(pSrc, pReq, 0, sUrl, sizeof(sUrl)) > 0, "builds"); + CHECK(streq(sUrl, GB "?read.time.min=2026-04-01&read.time.max=2026-04-01T01%3A30&read.time.inter=60"), + "got %s", sUrl); + size_t uNeed = DasStreamSrc_url(pSrc, pReq, 0, sUrl, 20); + CHECK(uNeed > 20 && strlen(sUrl) == 19, "snprintf style on a short buffer, need %zu", uNeed); + CHECK(DasStreamSrc_url(pSrc, pReq, 1, sUrl, sizeof(sUrl)) > 0 && + strncmp(sUrl, "http://127.0.0.1:8765/data/geomag_alt.d2s?", 42) == 0, "second mirror used"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasStreamSrc_url(pSrc, pReq, 2, sUrl, sizeof(sUrl)) == 0, "no third mirror"); + daslog_setlevel(DASLOG_WARN); + del_DasReq(pReq); + } + + /* The URL cases das_fetch has always been held to */ + URL_IS(pSrc, -1, K("time=2026-04-01,2026-04-01T01:30,60", "lshell"), NONE, + GB "?read.time.min=2026-04-01&read.time.max=2026-04-01T01%3A30&read.time.inter=60&read.data=lshell"); + URL_IS(pSrc, -1, K("coord.time.minimum=2026-02-01", "time.maximum=2026-02-02", "inter=60"), NONE, + GB "?read.time.min=2026-02-01&read.time.max=2026-02-02&read.time.inter=60"); + URL_IS(pSrc, 0, NONE, NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60"); + URL_IS(pSrc, 0, K("time.inter=30"), NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30"); + URL_IS(pSrc, 0, K("lshell=no"), NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60"); + URL_IS(pSrc, 0, K("csv", "delimiter=Tab", "csv.headers=false", "serial=text"), NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60&format.serial=text&format.type=csv&format.headers=1&format.delim=tab"); + URL_IS(pSrc, 0, K("data.lshell", "coord.time.inter=30"), NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30&read.data=lshell"); + const char* aRaw[] = {"read.time.inter", "5", "extra", "1", NULL}; + { + char sUrl[2048]; const char* aK[] = {NULL}; + CHECK(urlFor(pSrc, 0, aK, aRaw, sUrl, sizeof(sUrl)) && + streq(sUrl, GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=5&extra=1"), + "raw parameters win and add, got %s", sUrl); + } + /* the span form */ + URL_IS(pSrc, -1, K("time=2026-01-02T14", "inter=60"), NONE, + GB "?read.time.min=2026-01-02T14%3A00%3A00&read.time.max=2026-01-02T15%3A00%3A00&read.time.inter=60"); + + URL_FAILS(pSrc, -1, K("time=2026-04-01,2026-04-01T01:30")); /* required inter missing */ + URL_FAILS(pSrc, 0, K("foo=1")); /* no such key */ + URL_FAILS(pSrc, 0, K("fracSecs=4")); /* ambiguous */ + URL_FAILS(pSrc, 0, K("formats.xml")); /* no such format */ + URL_FAILS(pSrc, 0, K("lshell=maybe")); /* not a boolean */ + URL_FAILS(pSrc, 0, K("delimiter=Colon")); /* not in enum */ + URL_FAILS(pSrc, 0, K("version=das3")); /* geomag offers only das2 */ + URL_FAILS(pSrc, 0, K("data.nope")); /* no such variable */ + URL_FAILS(pSrc, 0, K("time.units=TAI")); /* fixed property */ + URL_FAILS(pSrc, 0, K("time=2026-01-02,2026-01-03,4,5")); /* too many range items */ + + /* mime follows the format choice */ + pReq = DasSrc_newReq(pSrc, 0); + CHECK(streq(DasSrc_mime(pSrc, NULL), "application/vnd.das.stream") || + strstr(DasSrc_mime(pSrc, NULL), "das") != NULL, "default is a das stream: %s", DasSrc_mime(pSrc, NULL)); + DasReq_set(pReq, "csv", "true"); + CHECK(streq(DasSrc_mime(pSrc, pReq), "text/csv"), "csv chosen: %s", DasSrc_mime(pSrc, pReq)); + del_DasReq(pReq); + + /* A requisition for some other source is refused */ + pReq = new_DasReq(); + DasReq_setSrc(pReq, "tag:das2.org,2012:site:/elsewhere", "", ""); + DasReq_set(pReq, "time", "2026-01-02"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasSrc_check(pSrc, pReq, NULL) == DASERR_SRC, "stamp mismatch refused"); + daslog_setlevel(DASLOG_WARN); + CHECK(DasReq_verdict(pReq, 0) == das_req_unchecked, "and nothing graded"); + + /* The same URI from another mirror, or a local copy, is the same source */ + DasReq_set(pReq, "inter", "60"); + DasReq_setSrc(pReq, DasSrc_path(pSrc), "file:///somewhere/else/geomag.json", ""); + CHECK(DasSrc_check(pSrc, pReq, NULL) == DAS_OKAY, "the URL is not identity"); + DasReq_setSrc(pReq, "", "file:///somewhere/else/geomag.json", ""); + CHECK(DasSrc_check(pSrc, pReq, NULL) == DAS_OKAY, "URL only is unstamped"); + del_DasReq(pReq); + + /* A source built with a shorthand path stamps the full URI */ + size_t uLen = 0; + char* sDoc = readFile(GEOMAG, &uLen); + DasSrc* pShort = sDoc ? new_DasSrc_json(sDoc, uLen, "site:uiowa/tracers/tracers-2/spice/geomagcoords/flex", NULL) : NULL; + free(sDoc); + CHECK(pShort && streq(DasSrc_path(pShort), DasSrc_path(pSrc)), "shorthand expanded, slash supplied: %s", + pShort ? DasSrc_path(pShort) : "(null)"); + if(pShort){ + pReq = DasSrc_newReq(pShort, 0); + CHECK(pReq && (DasSrc_check(pSrc, pReq, NULL) == DAS_OKAY), "its requisitions grade here"); + del_DasReq(pReq); + del_DasSrc(pShort); + } + + del_DasSrc(pSrc); +} + +static void test_hsk(void) +{ + printf("hsk, current layout at scale:\n"); + DasSrc* pSrc = loadSrc(HSK, NULL); + CHECK(pSrc != NULL, "load"); + if(pSrc == NULL) return; + + CHECK(ctrlCount(pSrc, -1) == 90, "3 + 73 + 14 controls, got %zu", ctrlCount(pSrc, -1)); + CHECK(ctrlCount(pSrc, das_src_sec_data) == 73, "73 variable toggles"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_data) == 73, "73 variable groups"); + CHECK(DasSrc_needsAuth(pSrc), "restricted"); + CHECK(DasSrc_contactCount(pSrc) >= 1, "has a contact"); + const char* sEmail = NULL; + const char* sName = DasSrc_contact(pSrc, 0, &sEmail); + CHECK(sName && sName[0] && sEmail, "contact name %s", sName ? sName : "(none)"); + CHECK(DasSrc_contact(pSrc, 99, NULL) == NULL, "past the contacts"); + CHECK(streq(DasSrc_path(pSrc), ""), "no path given, so no URI: %s", DasSrc_path(pSrc)); + + /* Every property has a key, every control a parameter, all titled */ + bool bAllKeyed = true, bAllParam = true, bAllTitled = true; + char sBuf[64]; + for(size_t u = 0; u < DasSrc_propCount(pSrc); ++u){ + const DasSrcProp* p = DasSrc_prop(pSrc, u); + if(DasSrcProp_key(p)[0] == '\0') bAllKeyed = false; + if(DasSrcProp_ctrl(p) && (DasSrcCtrl_param(DasSrcProp_ctrl(p)) == NULL)) bAllParam = false; + if(DasSrcProp_title(p, sBuf, sizeof(sBuf)) == NULL) bAllTitled = false; + } + CHECK(bAllKeyed && bAllParam && bAllTitled, "all keyed, parametered, titled"); + + /* groups are index ranges into one array, so a group's props are adjacent */ + const DasSrcGroup* pG = DasSrc_group(pSrc, das_src_sec_data, 5); + CHECK(pG && (DasSrcGroup_prop(pG, 0) == DasSrcProp_group(DasSrcGroup_prop(pG, 0))->pSrc->aProp + pG->uFirst), + "group indexing"); + + const DasSrcCtrl* pC = ctrlByKey(pSrc, "coords.time.min"); + CHECK(pC != NULL, "coords.time.min"); + if(pC){ + const char* sLo = NULL; const char* sHi = NULL; + CHECK(DasSrcCtrl_range(pC, &sLo, &sHi), "coordinate validRange reaches min"); + CHECK(sLo && (strncmp(sLo, "2025-", 5) == 0), "range low is the mission start, got %s", sLo); + CHECK(streq(DasSrcProp_title(DasSrcCtrl_prop(pC), sBuf, sizeof(sBuf)), "Minimum time value to stream"), + "node title wins over the composed one"); + CHECK(streq(DasSrcProp_units(DasSrcCtrl_prop(pC)), "UTC"), "UTC from the group units"); + } + pC = ctrlByKey(pSrc, "coords.time.res"); + CHECK(pC && !DasSrcCtrl_range(pC, NULL, NULL) && !DasSrcCtrl_required(pC), "res: no range, optional"); + const DasSrcProp* p = byKey(pSrc, "formats.csv.headers"); + CHECK(p && streq(DasSrcProp_value(p), "true") && (DasSrcProp_type(p) == das_srcprop_bool), "headers default true"); + + /* format groups: das is on by default, mime from it */ + pG = DasSrc_findGroup(pSrc, das_src_sec_format, "das"); + CHECK(pG && DasSrcGroup_mimeCount(pG) == 3, "three das mimes"); + CHECK(streq(DasSrc_mime(pSrc, NULL), "application/vnd.das2.das2stream"), "default mime: %s", DasSrc_mime(pSrc, NULL)); + + URL_IS(pSrc, 0, NONE, NONE, + HB "?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24"); + URL_IS(pSrc, 0, K("version=das2", "nmi_errors", "irq3_errors", "data.hsk_task_errors"), NONE, + HB "?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24&read.data=nmi_errors%2Cirq3_errors%2Chsk_task_errors&format.version=2"); + URL_IS(pSrc, 1, NONE, NONE, + HB "?read.time.min=2026-01-05T14%3A24&read.time.max=2026-01-06T14%3A24&format.type=cdf"); + URL_IS(pSrc, 0, K("formats.csv", "das.version=das2", "time.res=10"), NONE, + HB "?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24&bin.time.max=10&format.version=2&format.type=csv"); + + DasReq* pReq = DasSrc_newReq(pSrc, 1); + CHECK(pReq && streq(DasSrc_mime(pSrc, pReq), "application/x-cdf"), "cdf example means a cdf: %s", + pReq ? DasSrc_mime(pSrc, pReq) : ""); + del_DasReq(pReq); + + del_DasSrc(pSrc); +} + +static void test_juno(void) +{ + printf("juno_survey, 2018 layout:\n"); + DasSrc* pSrc = loadSrc(JUNO, "tag:das2.org,2012:site:/uiowa/juno/wav/survey/das2"); + CHECK(pSrc != NULL, "load"); + if(pSrc == NULL) return; + + CHECK(ctrlCount(pSrc, das_src_sec_coord) == 3, "three time controls"); + CHECK(ctrlCount(pSrc, das_src_sec_option) >= 8, "flag options found"); + CHECK(ctrlCount(pSrc, das_src_sec_data) == 0, "nothing settable in data"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_data) > 0, "but the variables are listed"); + CHECK(DasSrc_groupCount(pSrc, das_src_sec_option) == 1, "one options group"); + CHECK(DasSrc_findGroup(pSrc, das_src_sec_option, "") != NULL, "named by the empty string"); + + /* the fixed frequency axis is visible before any download */ + const DasSrcGroup* pG = DasSrc_findGroup(pSrc, das_src_sec_coord, "frequency"); + CHECK(pG != NULL, "frequency group"); + if(pG){ + CHECK(DasSrcGroup_ctrlCount(pG) == 0, "nothing to set"); + const DasSrcProp* pU = DasSrcGroup_findProp(pG, "units"); + CHECK(pU && (DasSrcProp_ctrl(pU) == NULL) && DasSrcProp_value(pU)[0], "units %s", + pU ? DasSrcProp_value(pU) : "(none)"); + } + /* time first even though the node lists frequency first */ + CHECK(streq(DasSrcGroup_name(DasSrc_group(pSrc, das_src_sec_coord, 0)), "time"), "time first"); + + /* Keys use the node's long spelling, matching folds it */ + const DasSrcProp* p = byKey(pSrc, "coords.time.minimum"); + CHECK(p != NULL, "coords.time.minimum keyed as spelled"); + CHECK(byKey(pSrc, "coords.time.min") == NULL, "not rewritten to min"); + if(p){ + const DasSrcCtrl* pC = DasSrcProp_ctrl(p); + CHECK(pC && streq(DasSrcCtrl_param(pC), "start_time"), "param"); + CHECK(pC && DasSrcCtrl_required(pC), "required from the set block"); + CHECK(streq(DasSrcProp_value(p), "2018-10-02T11:00"), "default kept"); + const char* sLo = NULL; const char* sHi = NULL; + CHECK(pC && DasSrcCtrl_range(pC, &sLo, &sHi) && streq(sHi, "now"), "set range to now"); + CHECK(DasSrcProp_type(p) == das_srcprop_time, "type from the old parameter dict"); + CHECK(streq(DasSrcProp_units(p), "UTC"), "UTC from the old units dict"); + char sBuf[64]; + CHECK(streq(DasSrcProp_title(p, sBuf, sizeof(sBuf)), "Minimum Time value to output"), + "composed from the group name, got '%s'", DasSrcProp_title(p, sBuf, sizeof(sBuf))); + } + p = byKey(pSrc, "coords.time.resolution"); + CHECK(p && streq(DasSrcProp_value(p), "86.4"), "number default in source notation"); + CHECK(p && streq(DasSrcProp_units(p), "s"), "units string on the property"); + + p = byKey(pSrc, "options.lfrl"); + CHECK(p != NULL, "options.lfrl"); + if(p){ + CHECK(streq(DasSrcGroup_name(DasSrcProp_group(p)), ""), "options have no group name"); + CHECK(DasSrcProp_type(p) == das_srcprop_bool, "flag toggle is boolean"); + CHECK(streq(DasSrcCtrl_param(DasSrcProp_ctrl(p)), "params"), "packed parameter"); + CHECK(streq(DasSrcProp_value(p), "false"), "default false"); + } + p = byKey(pSrc, "options.das2_text"); + CHECK(p && (DasSrcProp_type(p) == das_srcprop_bool), "boolean from a boolean default"); + + /* 2018 examples come out in interface terms */ + CHECK(DasSrc_exampleCount(pSrc) >= 8, "examples, got %zu", DasSrc_exampleCount(pSrc)); + CHECK(streq(DasSrc_exampleLabel(pSrc, 0), "Perijove 16"), "label from the title: %s", DasSrc_exampleLabel(pSrc, 0)); + DasReq* pReq = DasSrc_newReq(pSrc, 0); + CHECK(pReq != NULL, "example 0 req"); + if(pReq){ + CHECK(streq(DasReq_value(pReq, "coords.time.minimum"), "2018-10-02T11:00"), "start_time mapped back"); + CHECK(streq(DasReq_value(pReq, "coords.time.resolution"), "86.4"), "resolution mapped back"); + CHECK(DasReq_rawCount(pReq) == 0, "nothing left raw, %zu", DasReq_rawCount(pReq)); + del_DasReq(pReq); + } + pReq = DasSrc_newReq(pSrc, 7); + CHECK(pReq != NULL, "example 7 req"); + if(pReq){ + CHECK(streq(DasReq_value(pReq, "options.b"), "true"), "params=B became the b toggle"); + CHECK(DasReq_rawCount(pReq) == 0, "nothing left raw"); + del_DasReq(pReq); + } + + URL_IS(pSrc, -1, K("time=2021-02-21,2021-02-22", "b", "lfrl", "das2_text"), NONE, + JB "&end_time=2021-02-22&start_time=2021-02-21¶ms=B%20LFRL&ascii=true"); + URL_IS(pSrc, -1, K("time=2021-02-21,2021-02-22,0", "ascii", "option.xmas=true"), NONE, + JB "&end_time=2021-02-22&start_time=2021-02-21&resolution=0¶ms=ASCII%20XMAS"); + /* example 7 with the resolution changed. The parameter order is the + interface order now that the example is settings, not raw text. */ + URL_IS(pSrc, 7, K("resolution=10"), NONE, + JB "&end_time=2016-177T00%3A00&start_time=2016-176T00%3A00&resolution=10¶ms=B"); + + del_DasSrc(pSrc); +} + +static void test_req(void) +{ + printf("Requisitions without a source:\n"); + DasReq* pReq = new_DasReq(); + CHECK(pReq != NULL, "new"); + CHECK(DasReq_count(pReq) == 0 && streq(DasReq_srcPath(pReq), ""), "empty and unstamped"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasReq_set(pReq, "", "x") == DASERR_SRC, "empty key refused"); + daslog_setlevel(DASLOG_WARN); + CHECK(DasReq_set(pReq, "Time", "2026-01-02") == DAS_OKAY, "set"); + CHECK(DasReq_set(pReq, "time", "2026-01-03") == DAS_OKAY, "replace, case insensitive"); + CHECK(DasReq_count(pReq) == 1 && streq(DasReq_value(pReq, "TIME"), "2026-01-03"), "one entry"); + CHECK(DasReq_set(pReq, "res", NULL) == DAS_OKAY && streq(DasReq_value(pReq, "res"), ""), "NULL is empty"); + CHECK(DasReq_find(pReq, "res") == 1 && DasReq_find(pReq, "nope") == -1, "find"); + CHECK(DasReq_verdict(pReq, 0) == das_req_unchecked && streq(DasReq_why(pReq, 0), ""), "unchecked"); + CHECK(DasReq_setRaw(pReq, "extra", "1") == DAS_OKAY && DasReq_rawCount(pReq) == 1, "raw"); + const char* sVal = NULL; + CHECK(streq(DasReq_raw(pReq, 0, &sVal), "extra") && streq(sVal, "1"), "raw read back"); + CHECK(DasReq_remove(pReq, "res") && !DasReq_remove(pReq, "res"), "remove once"); + DasReq_setLabel(pReq, "a \"quoted\" label\n"); + DasReq_setSrc(pReq, "site:/x", "http://x/y.json", "0.7"); + + /* round trip through text */ + char* sText = DasReq_text(pReq); + CHECK(sText != NULL && strstr(sText, "\\\"quoted\\\"") != NULL, "escaped label"); + DasReq* pBack = new_DasReq_text(sText, strlen(sText)); + CHECK(pBack != NULL, "parses back"); + if(pBack){ + CHECK(streq(DasReq_label(pBack), "a \"quoted\" label\n"), "label survives"); + CHECK(streq(DasReq_srcPath(pBack), "site:/x") && streq(DasReq_srcVersion(pBack), "0.7"), "stamp survives"); + CHECK(streq(DasReq_value(pBack, "time"), "2026-01-03"), "setting survives"); + CHECK(DasReq_rawCount(pBack) == 1, "raw survives"); + del_DasReq(pBack); + } + free(sText); + + /* the example settings shape reads too */ + const char* sEx = "{\"settings\":{\"coords/time/props/min\":\"2026-01-01\",\"coords/time/props/max\":\"2026-01-02\"}," + "\"label\":\"ex\"}"; + pBack = new_DasReq_text(sEx, strlen(sEx)); + CHECK(pBack && streq(DasReq_value(pBack, "coords.time.min"), "2026-01-01"), "slash paths fold to keys"); + del_DasReq(pBack); + + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasReq_text("[1,2]", 5) == NULL, "not an object"); + CHECK(new_DasReq_file("test/das_fetch/no_such_file.json") == NULL, "missing file"); + daslog_setlevel(DASLOG_WARN); + + /* file round trip and a copy. ./tmp is git ignored, so a fresh + checkout, CI for one, does not have it */ + das_mkdirsto("tmp/TestSource_req.json"); + CHECK(DasReq_write(pReq, "tmp/TestSource_req.json") == DAS_OKAY, "write"); + pBack = new_DasReq_file("tmp/TestSource_req.json"); + CHECK(pBack && streq(DasReq_value(pBack, "time"), "2026-01-03"), "read back from file"); + del_DasReq(pBack); + remove("tmp/TestSource_req.json"); + + DasReq* pCopy = copy_DasReq(pReq); + CHECK(pCopy && DasReq_count(pCopy) == 1 && DasReq_rawCount(pCopy) == 1, "copy"); + del_DasReq(pCopy); + + DasReq_clear(pReq); + CHECK(DasReq_count(pReq) == 0 && DasReq_rawCount(pReq) == 0 && streq(DasReq_srcPath(pReq), "site:/x"), + "clear keeps the stamp"); + del_DasReq(pReq); +} + +static void test_node(void) +{ + printf("From a node tree:\n"); + DasCat* pCat = new_DasCat(NULL, "TestSource"); + DasCat_setSeed(pCat, GEOMAG, "tag:das2.org,2012:site:/uiowa/geomag/flex"); + DasNode* pRoot = DasCat_seed(pCat); + CHECK(pRoot != NULL, "node loads"); + if(pRoot == NULL){ del_DasCat(pCat); return; } + DasSrc* pSrc = new_DasSrc_node(pRoot); + CHECK(pSrc != NULL, "source from node"); + del_DasCat(pCat); /* takes the node; the source must not care */ + if(pSrc){ + CHECK(streq(DasSrc_path(pSrc), "tag:das2.org,2012:site:/uiowa/geomag/flex"), "path from the node"); + CHECK(ctrlCount(pSrc, -1) == 17, "same interface after the tree is gone"); + URL_IS(pSrc, 0, NONE, NONE, + GB "?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60"); + del_DasSrc(pSrc); + } + + daslog_setlevel(DASLOG_CRIT); + const char* sNotSrc = "{\"type\":\"Catalog\",\"catalog\":{}}"; + CHECK(new_DasSrc_json(sNotSrc, strlen(sNotSrc), NULL, NULL) == NULL, "a catalog is not a source"); + CHECK(new_DasSrc_json("not json", 8, NULL, NULL) == NULL, "garbage is not a source"); + daslog_setlevel(DASLOG_WARN); + + /* a set block with no param is a fixed property with a warning, the + rest still builds */ + const char* sDoc = + "{\"type\":\"HttpStreamSrc\",\"protocol\":{\"baseUrls\":[\"http://x/y\"],\"httpParams\":{}}," + "\"interface\":{\"options\":{" + " \"good\":{\"value\":false,\"set\":{\"param\":\"g\",\"value\":true}}," + " \"broken\":{\"value\":false,\"set\":{\"value\":true}}," + " \"doc_only\":{\"value\":\"x\"}" + "}}}"; + daslog_setlevel(DASLOG_ERROR); + DasSrc* pSrc2 = new_DasSrc_json(sDoc, strlen(sDoc), NULL, NULL); + daslog_setlevel(DASLOG_WARN); + CHECK(pSrc2 != NULL, "builds"); + if(pSrc2){ + CHECK(streq(DasSrc_path(pSrc2), ""), "nameless and placeless means no URI: %s", DasSrc_path(pSrc2)); + CHECK(DasSrc_propCount(pSrc2) == 3 && ctrlCount(pSrc2, -1) == 1, "three props, one control"); + CHECK(streq(DasSrcProp_key(DasSrc_prop(pSrc2, 0)), "options.good"), "the good one"); + CHECK(DasSrcProp_type(DasSrc_prop(pSrc2, 0)) == das_srcprop_bool, "boolean by default value"); + CHECK(streq(DasSrcProp_title(DasSrc_prop(pSrc2, 0), NULL, 0), ""), "untitled option is empty"); + CHECK(streq(DasSrc_mime(pSrc2, NULL), ""), "no mime stated"); + URL_IS(pSrc2, -1, K("good"), NONE, "http://x/y?g=true"); + URL_IS(pSrc2, -1, NONE, NONE, "http://x/y"); + del_DasSrc(pSrc2); + } +} + +/* Choosing among a container's sources by what the client reads */ +static void test_pick(void) +{ + printf("Loadable sources under a container:\n"); + const char* aNames[4]; + DasCat* pCat = new_DasCat(NULL, "TestSource"); + DasCat_setSeed(pCat, "test/das_fetch/cat/twin.json", "site:/uiowa/twin"); + DasNode* pTwin = DasCat_seed(pCat); + CHECK(pTwin != NULL, "twin loads"); + if(pTwin == NULL){ del_DasCat(pCat); return; } + CHECK(dassrc_loadable(pTwin, aNames, 4) == 2 && streq(aNames[0], "flex") && streq(aNames[1], "das3"), + "two loadable children in catalog order, the DSDF is skipped"); + CHECK(dassrc_loadable(pTwin, NULL, 0) == 2, "counting only"); + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasSrc_node(pTwin) == NULL, "a container is not a source, no child is picked"); + daslog_setlevel(DASLOG_WARN); + + DasNode* pSub = DasCat_subNode(pCat, pTwin, "das3"); + CHECK(pSub != NULL, "the caller's choice loads through the tree"); + CHECK(dassrc_loadable(pSub, NULL, 0) == 0, "a source has no loadable children"); + DasSrc* pSrc = (pSub == NULL) ? NULL : new_DasSrc_node(pSub); + CHECK(pSrc != NULL && streq(DasSrc_path(pSrc), "tag:das2.org,2012:site:/uiowa/twin/das3"), "source from the chosen child"); + del_DasSrc(pSrc); + del_DasCat(pCat); + + pCat = new_DasCat(NULL, "TestSource"); + DasCat_setSeed(pCat, "test/das_fetch/cat/old.json", "site:/uiowa/old"); + DasNode* pOld = DasCat_seed(pCat); + CHECK(pOld != NULL, "old loads"); + if(pOld == NULL){ del_DasCat(pCat); return; } + CHECK(dassrc_loadable(pOld, aNames, 4) == 2, "a 2018 Collection lists both HttpStreamSrc children"); + + /* Presented in the current layout: the provides list each convention + implies is there, the convention is untouched, and the DOM still + serializes with the additions in place */ + CHECK(streq(DasNode_subName(pOld, 0), "das2") && streq(DasNode_subName(pOld, 1), "hapi"), "fixture order"); + const char* sMime = DasJdo_string(DasJdo_get(DasNode_subStub(pOld, 0), "provides/0")); + CHECK(sMime != NULL && streq(sMime, "application/vnd.das2.das2stream"), "das2/2.2 presents as a das2 stream"); + sMime = DasJdo_string(DasJdo_get(DasNode_subStub(pOld, 1), "provides/0")); + CHECK(sMime != NULL && streq(sMime, "application/vnd.hapi.stream"), "hapi/2.0 presents as a hapi stream"); + CHECK(DasJdo_get(DasNode_subStub(pOld, 1), "convention") != NULL, "the convention stays"); + size_t uDoc = 0; + char* sDoc = (char*)DasJdo_writeMinified(DasNode_getJdo(pOld, NULL), &uDoc); + CHECK(sDoc != NULL && strstr(sDoc, "\"provides\":[\"application/vnd.hapi.stream\"]") != NULL, "the added list serializes"); + free(sDoc); + del_DasCat(pCat); +} + +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + test_grammar(); + test_scalar(); + test_helpers(); + test_geomag(); + test_hsk(); + test_juno(); + test_req(); + test_node(); + test_pick(); + printf("%d checks passed, %d failed\n", g_nPass, g_nFail); + return g_nFail ? 1 : 0; +} diff --git a/test/TestVar.c b/test/TestVar.c index 8842ef2..42dc7eb 100644 --- a/test/TestVar.c +++ b/test/TestVar.c @@ -288,7 +288,7 @@ static int test_byte_runs(void) MUST(DasVar_get((DasVar*)pStr, aLoc, DAS_BS_NULL, &dm) == 0); /* sVal is read below */ CHECK(dm.vt == vtText); const char* sVal = NULL; - memcpy(&sVal, &dm, sizeof(const char*)); + memcpy((void*)&sVal, &dm, sizeof(const char*)); /* quiet msvc */ MUST(sVal != NULL); CHECK(strcmp(sVal, "BURST") == 0); diff --git a/test/TestWalk.c b/test/TestWalk.c new file mode 100644 index 0000000..1340763 --- /dev/null +++ b/test/TestWalk.c @@ -0,0 +1,413 @@ +/** @file TestWalk.c Unit tests for DasNodeWalk, driven by hand from + * fixture files. No network, no DasCat. */ + +/* Author: Chris Piker , via Claude Fable 5.1 + * + * This file is intended to demonstrate an interface. This is free + * and unencumbered software released into the public domain + * + * Anyone is free to copy, modify, publish, use, compile, sell, or + * distribute this file, either in source code form or as a compiled + * binary, for any purpose, commercial or non-commercial, and by any + * means. + * + * In jurisdictions that recognize copyright laws, the author or authors + * of this file dedicate any and all copyright interest in this file to + * the public domain. We make this dedication for the benefit of the + * public at large and to the detriment of our heirs and successors. We + * intend this dedication to be an overt act of relinquishment in + * perpetuity of all present and future rights to this file under + * copyright law. + * + * THIS FILE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + * + * For more information, please refer to + */ + +/* Runs from the top of the source tree, the fixture node files in + * test/das_walk/ refer to each other by paths from there. This program + * is the walk's driver, which is what a browser build is: it reads the + * file a walk asks for and feeds it back, or reports that it could not. + * Because the driver is in the test, the fallback paths that only a + * misbehaving server would exercise are exercised here on purpose: a + * missing mirror, a look-alike child, a document that is not JSON, a + * child with nowhere to load from. */ + +#include +#include +#include +#include + +#include +#include + +static int g_nPass = 0; +static int g_nFail = 0; + +#define CHECK(COND, ...) do{ \ + if(COND){ ++g_nPass; } \ + else{ ++g_nFail; printf(" FAIL %s:%d: ", __FILE__, __LINE__); printf(__VA_ARGS__); putchar('\n'); } \ +}while(0) + +static bool streq(const char* a, const char* b){ return (a != NULL) && (b != NULL) && (strcmp(a, b) == 0); } + +#define ROOT_URL "test/das_walk/root.json" +#define ROOT_URI "tag:das2.org,2012:site:/walk" + +/* ************************************************************************* */ +/* The driver: files in, documents out */ + +static char* readFile(const char* sPath, size_t* pLen) +{ + FILE* pFile = fopen(sPath, "rb"); + if(pFile == NULL) return NULL; + char* pBuf = NULL; + size_t uCap = 0, uLen = 0; + char sChunk[4096]; + size_t uRead; + while((uRead = fread(sChunk, 1, sizeof(sChunk), pFile)) > 0){ + if(uLen + uRead > uCap){ + uCap = (uLen + uRead) * 2; + pBuf = (char*)realloc(pBuf, uCap); + } + memcpy(pBuf + uLen, sChunk, uRead); + uLen += uRead; + } + fclose(pFile); + *pLen = uLen; + return pBuf; +} + +/* Records every URL asked for, in order, so a test can say what a walk + * should have tried */ +#define MAX_LOG 16 +struct drive_log { + const char* aUrl[MAX_LOG]; + size_t uUrls; +}; + +/* Answer one step with candidate uIdx: feed the file or report it missing */ +static DasErrCode step(DasNodeWalk* pWalk, size_t uIdx, struct drive_log* pLog) +{ + const char* sUrl = DasNodeWalk_url(pWalk, uIdx); + if((pLog != NULL) && (pLog->uUrls < MAX_LOG)) pLog->aUrl[pLog->uUrls++] = sUrl; + size_t uLen = 0; + char* pDoc = readFile(sUrl, &uLen); + if(pDoc == NULL){ + DasNodeWalk_fail(pWalk, uIdx, "no such file"); + return DASERR_NOTIMP; /* any code that is not DAS_OKAY or DASERR_NODE */ + } + DasErrCode rc = DasNodeWalk_feed(pWalk, uIdx, pDoc, uLen, NULL); + free(pDoc); + return rc; +} + +static DasNode* drive(DasNodeWalk* pWalk, struct drive_log* pLog) +{ + if(pLog != NULL) pLog->uUrls = 0; + while(DasNodeWalk_state(pWalk) == das_walk_fetch) step(pWalk, 0, pLog); + return DasNodeWalk_result(pWalk); +} + +static bool logIs(const struct drive_log* pLog, size_t u, ...) +{ + if(pLog->uUrls != u) return false; + va_list ap; + va_start(ap, u); + bool bOk = true; + for(size_t i = 0; i < u; ++i){ + const char* s = va_arg(ap, const char*); + if(!streq(pLog->aUrl[i], s)) bOk = false; + } + va_end(ap); + return bOk; +} + +static void showLog(const struct drive_log* pLog) +{ + printf(" asked for:"); + for(size_t i = 0; i < pLog->uUrls; ++i) printf(" %s", pLog->aUrl[i]); + putchar('\n'); +} + +static DasNode* loadRoot(void) +{ + size_t uLen = 0; + char* pDoc = readFile(ROOT_URL, &uLen); + DasNode* pRoot = (pDoc == NULL) ? NULL : new_DasNode_json(pDoc, uLen, ROOT_URL, "site:/walk"); + free(pDoc); + return pRoot; +} + +/* ************************************************************************* */ +static void test_json(void) +{ + printf("TEST: nodes from bytes\n"); + DasNode* pRoot = loadRoot(); + CHECK(pRoot != NULL, "root loads from bytes"); + if(pRoot == NULL) return; + CHECK(DasNode_isCatalog(pRoot), "it is a catalog"); + CHECK(streq(DasNode_pathUri(pRoot), ROOT_URI), "shorthand expanded: %s", DasNode_pathUri(pRoot)); + CHECK(streq(DasNode_srcUrl(pRoot), ROOT_URL), "source URL kept"); + CHECK(DasNode_subCount(pRoot) == 5, "five children listed, none loaded"); + CHECK(DasNode_subType(pRoot, 4) == d2node_stream_src, "last child is a stream source"); + del_DasNode(pRoot); + + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasNode_json("not json", 8, "x", NULL) == NULL, "garbage is not a node"); + CHECK(new_DasNode_json("{\"type\":\"Nope\"}", 15, "x", NULL) == NULL, "unknown type is not a node"); + daslog_setlevel(DASLOG_WARN); + + const char* sDoc = "{\"type\":\"HttpStreamSrc\",\"protocol\":{}}"; + DasNode* pNode = new_DasNode_json(sDoc, strlen(sDoc), NULL, NULL); + CHECK(pNode && (DasNode_pathUri(pNode) == NULL), "nameless means no URI, a root"); + CHECK(pNode && streq(DasNode_srcUrl(pNode), ""), "no URL is the empty string"); + del_DasNode(pNode); + pNode = new_DasNode_json(sDoc, strlen(sDoc), "http://host/x.json", NULL); + CHECK(pNode && (DasNode_pathUri(pNode) == NULL), "a location is not a name either"); + CHECK(pNode && streq(DasNode_srcUrl(pNode), "http://host/x.json"), "but it is kept as the source URL"); + del_DasNode(pNode); +} + +/* ************************************************************************* */ +static void test_steps(void) +{ + printf("TEST: one step at a time\n"); + DasNode* pRoot = loadRoot(); + if(pRoot == NULL) return; + + DasNodeWalk* pWalk = new_DasNodeWalk(pRoot, "src"); + CHECK(pWalk != NULL, "walk starts"); + if(pWalk == NULL){ del_DasNode(pRoot); return; } + CHECK(DasNodeWalk_state(pWalk) == das_walk_fetch, "a document is wanted"); + CHECK(DasNodeWalk_urlCount(pWalk) == 1, "one candidate"); + CHECK(streq(DasNodeWalk_url(pWalk, 0), "test/das_walk/src.json"), "the child's URL"); + CHECK(DasNodeWalk_url(pWalk, 1) == NULL, "past the end is NULL"); + CHECK(streq(DasNodeWalk_uri(pWalk), ROOT_URI "/src"), "the child's URI: %s", DasNodeWalk_uri(pWalk)); + CHECK(DasNodeWalk_result(pWalk) == NULL, "no result yet"); + CHECK(DasNodeWalk_nearest(pWalk, NULL) == NULL, "nothing nearest while running"); + + daslog_setlevel(DASLOG_CRIT); + CHECK(DasNodeWalk_feed(pWalk, 1, "{}", 2, NULL) != DAS_OKAY, "feeding a candidate not on offer is refused"); + daslog_setlevel(DASLOG_WARN); + CHECK(DasNodeWalk_state(pWalk) == das_walk_fetch, "and changes nothing"); + + CHECK(step(pWalk, 0, NULL) == DAS_OKAY, "the source document parses"); + CHECK(DasNodeWalk_state(pWalk) == das_walk_done, "walk is done"); + DasNode* pSrc = DasNodeWalk_result(pWalk); + CHECK(pSrc != NULL, "result"); + CHECK(pSrc && DasNode_isStreamSrc(pSrc), "it is a stream source"); + CHECK(pSrc && streq(DasNode_pathUri(pSrc), ROOT_URI "/src"), "named by the walk"); + CHECK(pSrc && streq(DasNode_srcUrl(pSrc), "test/das_walk/src.json"), "loaded from the candidate"); + CHECK(DasNodeWalk_urlCount(pWalk) == 0, "nothing on offer when done"); + daslog_setlevel(DASLOG_CRIT); + CHECK(DasNodeWalk_feed(pWalk, 0, "{}", 2, NULL) != DAS_OKAY, "a finished walk takes nothing"); + daslog_setlevel(DASLOG_WARN); + del_DasNodeWalk(pWalk); + + /* The result was attached: the same path needs no document now */ + pWalk = new_DasNodeWalk(pRoot, "src"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_done), "second walk is done at once"); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pSrc), "and finds the same node"); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "/src"); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pSrc), "leading separator is accepted"); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, ""); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pRoot), "empty path is the start node"); + del_DasNodeWalk(pWalk); + + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasNodeWalk(pSrc, "deeper") == NULL, "no walk below a source"); + daslog_setlevel(DASLOG_WARN); + + del_DasNode(pSrc); /* a child: ignored, its parent owns it */ + CHECK(streq(DasNode_srcUrl(pSrc), "test/das_walk/src.json"), "a child is not deleted separately"); + del_DasNode(pRoot); +} + +/* ************************************************************************* */ +static void test_fallback(void) +{ + printf("TEST: mirrors, look-alikes and dead ends\n"); + struct drive_log log; + DasNode* pRoot = loadRoot(); + if(pRoot == NULL) return; + + /* The first mirror of "a" does not exist */ + DasNodeWalk* pWalk = new_DasNodeWalk(pRoot, "a/b"); + CHECK(pWalk && (DasNodeWalk_urlCount(pWalk) == 2), "two mirrors on offer"); + DasNode* pB = drive(pWalk, &log); + CHECK(pB != NULL, "a/b resolves through the second mirror"); + CHECK(logIs(&log, 3, "test/das_walk/missing.json", "test/das_walk/a.json", "test/das_walk/src.json"), + "missing mirror, good mirror, leaf"); + if(g_nFail) showLog(&log); + CHECK(pB && streq(DasNode_pathUri(pB), ROOT_URI "/a/b"), "URI composed through two levels: %s", + pB ? DasNode_pathUri(pB) : "(null)"); + del_DasNodeWalk(pWalk); + + /* Both levels attached: "a" costs nothing now, and so does a/b */ + pWalk = new_DasNodeWalk(pRoot, "a"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_done), "the intermediate node was kept"); + DasNode* pA = pWalk ? DasNodeWalk_result(pWalk) : NULL; + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "a/b"); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pB), "and so was the leaf"); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pA, "b"); + CHECK(pWalk && (DasNodeWalk_result(pWalk) == pB), "a walk may start at the intermediate node"); + del_DasNodeWalk(pWalk); + + /* "a" is a prefix of "ab", so it is tried first and rejected by the + separator rule; the loaded copy of "a" is consulted and the listing + entry for it skipped, so no mirror of "a" is asked for again */ + pWalk = new_DasNodeWalk(pRoot, "ab/x"); + DasNode* pX = drive(pWalk, &log); + CHECK(pX != NULL, "ab/x resolves"); + CHECK(logIs(&log, 2, "test/das_walk/ab.json", "test/das_walk/src.json"), "look-alike skipped via the loaded copy"); + if(g_nFail) showLog(&log); + del_DasNodeWalk(pWalk); + del_DasNode(pRoot); + + /* The same on a fresh tree: "a" has to be fetched to be rejected, and + since it led nowhere it is not kept */ + pRoot = loadRoot(); + pWalk = new_DasNodeWalk(pRoot, "ab/x"); + pX = drive(pWalk, &log); + CHECK(pX && streq(DasNode_pathUri(pX), ROOT_URI "/ab/x"), "ab/x on a fresh tree"); + CHECK(logIs(&log, 4, "test/das_walk/missing.json", "test/das_walk/a.json", "test/das_walk/ab.json", + "test/das_walk/src.json"), "a was tried first, through both mirrors, then ab"); + if(g_nFail) showLog(&log); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "a"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_fetch), "a dead end is not kept"); + del_DasNodeWalk(pWalk); + + /* A document that is not JSON: feed says so, the walk moves on */ + pWalk = new_DasNodeWalk(pRoot, "junk/x"); + daslog_setlevel(DASLOG_CRIT); + DasErrCode rc = step(pWalk, 0, NULL); + daslog_setlevel(DASLOG_WARN); + CHECK(rc == DASERR_NODE, "junk is reported to the driver"); + CHECK(DasNodeWalk_state(pWalk) == das_walk_failed, "and nothing else to try"); + const char* sRest = NULL; + CHECK(streq(DasNodeWalk_nearest(pWalk, &sRest), ROOT_URI) && streq(sRest, "junk/x"), + "nearest is the root with the whole path below it"); + del_DasNodeWalk(pWalk); + + /* Nothing listed, or listed with no URLs: failed before any step */ + daslog_setlevel(DASLOG_CRIT); + pWalk = new_DasNodeWalk(pRoot, "nothere"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_failed), "unknown child fails at once"); + CHECK(pWalk && (DasNodeWalk_urlCount(pWalk) == 0), "nothing on offer"); + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "nourls/x"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_failed), "a child with no URLs fails at once"); + del_DasNodeWalk(pWalk); + daslog_setlevel(DASLOG_WARN); + + /* A driver that fetched the mirrors in parallel feeds the one that + answered, skipping the ones before it */ + pWalk = new_DasNodeWalk(pRoot, "a/b"); + CHECK(pWalk && (DasNodeWalk_urlCount(pWalk) == 2), "two mirrors again on a fresh tree"); + CHECK(step(pWalk, 1, NULL) == DAS_OKAY, "the second mirror fed directly"); + CHECK(DasNodeWalk_state(pWalk) == das_walk_fetch, "on to the leaf"); + CHECK(streq(DasNodeWalk_uri(pWalk), ROOT_URI "/a/b"), "leaf URI"); + /* Abandon it here: the node loaded for "a" belongs to the walk and + must go with it, see valgrind */ + del_DasNodeWalk(pWalk); + pWalk = new_DasNodeWalk(pRoot, "a"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_fetch), "an abandoned walk attached nothing"); + del_DasNodeWalk(pWalk); + + del_DasNode(pRoot); +} + +/* ************************************************************************* */ +static void test_federated(void) +{ + printf("TEST: the compiled in seed\n"); + size_t uRoots = 0; + const char** aRoots = das_seed_urls(&uRoots); + + daslog_setlevel(DASLOG_CRIT); + CHECK(new_DasNodeWalk(NULL, "src") == NULL, "a walk needs a start node"); + daslog_setlevel(DASLOG_WARN); + + DasNodeWalk* pWalk = new_DasNodeWalk_seed(); + CHECK(pWalk != NULL, "seed walk starts"); + if(pWalk == NULL) return; + CHECK(DasNodeWalk_state(pWalk) == das_walk_fetch, "a seed document is wanted"); + CHECK(DasNodeWalk_urlCount(pWalk) == uRoots, "every seed URL is a candidate"); + CHECK(streq(DasNodeWalk_url(pWalk, 0), aRoots[0]), "in order"); + CHECK(streq(DasNodeWalk_uri(pWalk), ""), "a seed has no URI"); + + DasNodeWalk_fail(pWalk, 0, "no network in a unit test"); + CHECK(DasNodeWalk_urlCount(pWalk) == uRoots - 1, "one mirror fewer"); + CHECK(streq(DasNodeWalk_url(pWalk, 0), aRoots[1]), "the next mirror is first now"); + + /* A seed must be a Catalog; a source document is dropped */ + size_t uLen = 0; + char* pDoc = readFile("test/das_walk/src.json", &uLen); + daslog_setlevel(DASLOG_CRIT); + DasErrCode rc = DasNodeWalk_feed(pWalk, 0, pDoc, uLen, NULL); + daslog_setlevel(DASLOG_WARN); + free(pDoc); + CHECK(rc == DASERR_NODE, "a source is not a seed"); + CHECK(DasNodeWalk_state(pWalk) == das_walk_failed, "no mirrors left"); + CHECK(DasNodeWalk_nearest(pWalk, NULL) == NULL, "nothing nearest when no seed loaded"); + del_DasNodeWalk(pWalk); + + /* A stand in seed that lists the fixture tree as the site: node, so + its children are site:/a, site:/src and so on. The seed is the + caller's; everything else is a walk from it and stays under it. */ + pWalk = new_DasNodeWalk_seed(); + pDoc = readFile("test/das_walk/fedroot.json", &uLen); + CHECK(DasNodeWalk_feed(pWalk, 0, pDoc, uLen, "https://mirror/index.json") == DAS_OKAY, "a stand in seed"); + free(pDoc); + CHECK(DasNodeWalk_state(pWalk) == das_walk_done, "the seed is the result"); + DasNode* pRoot = DasNodeWalk_result(pWalk); + del_DasNodeWalk(pWalk); + CHECK(pRoot && DasNode_isCatalog(pRoot), "the seed outlives the walk"); + CHECK(pRoot && (DasNode_pathUri(pRoot) == NULL), "a seed with no prefix is a root, the one node with no URI"); + CHECK(pRoot && streq(DasNode_srcUrl(pRoot), aRoots[0]), "loaded from the first mirror, this walk never failed one"); + if(pRoot == NULL) return; + + pWalk = new_DasNodeWalk(pRoot, "tag:das2.org,2012:site:/src"); + CHECK(pWalk && (DasNodeWalk_state(pWalk) == das_walk_fetch), "a full URI walks from the seed"); + CHECK(pWalk && streq(DasNodeWalk_uri(pWalk), "tag:das2.org,2012:site:"), "first level named by the seed's prefix child: %s", + pWalk ? DasNodeWalk_uri(pWalk) : "(null)"); + CHECK(pWalk && streq(DasNodeWalk_url(pWalk, 0), ROOT_URL), "the fixture root stands in for site"); + DasNode* pSrc = pWalk ? drive(pWalk, NULL) : NULL; + CHECK(pSrc && streq(DasNode_pathUri(pSrc), "tag:das2.org,2012:site:/src"), "full URI: %s", + pSrc ? DasNode_pathUri(pSrc) : "(null)"); + del_DasNodeWalk(pWalk); + + /* A sibling shares the levels already walked */ + struct drive_log log; + pWalk = new_DasNodeWalk(pRoot, "tag:das2.org,2012:site:/a/b"); + DasNode* pB = pWalk ? drive(pWalk, &log) : NULL; + CHECK(pB != NULL, "a sibling path resolves"); + CHECK(logIs(&log, 3, "test/das_walk/missing.json", "test/das_walk/a.json", "test/das_walk/src.json"), + "the site level was not fetched again"); + if(g_nFail) showLog(&log); + del_DasNodeWalk(pWalk); + + del_DasNode(pRoot); /* takes site, a, b and src with it */ +} + +/* ************************************************************************* */ +int main(int argc, char** argv) +{ + das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + + test_json(); + test_steps(); + test_fallback(); + test_federated(); + + printf("%d checks passed, %d failed\n", g_nPass, g_nFail); + return (g_nFail == 0) ? 0 : 1; +} diff --git a/test/das_cache_srv.py b/test/das_cache_srv.py new file mode 100755 index 0000000..0bb5027 --- /dev/null +++ b/test/das_cache_srv.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Throwaway HTTP file server for test/das_cache_test.sh. + +Serves the files in one directory the way a well behaved static server +does: ETag and Last-Modified on every 200, and 304 Not Modified when the +request's If-None-Match matches or its If-Modified-Since is not older than +the file. With --no-validators it behaves like a server that knows nothing +of caching, which is what dasflex does today. Every request is appended +to the log file as one line, so the shell test can see what the client +sent and what it got back. + +Drafted by Claude Fable 5.1, filed by Chris Piker. Python 3.6 compatible. +""" + +import argparse +import email.utils +import os +import sys +from http.server import BaseHTTPRequestHandler, HTTPServer + +g_dir = "." +g_log = None +g_validators = True + + +class Handler(BaseHTTPRequestHandler): + + def log_message(self, fmt, *args): + pass # the request log below is the one the test reads + + def _log(self, code, inm, ims): + with open(g_log, "a") as f: + f.write("GET %s %d inm=%s ims=%s\n" % ( + self.path, code, inm if inm else "-", ims if ims else "-" + )) + + def do_GET(self): + sPath = os.path.join(g_dir, self.path.lstrip("/").split("?")[0]) + inm = self.headers.get("If-None-Match") + ims = self.headers.get("If-Modified-Since") + + if not os.path.isfile(sPath): + self.send_response(404) + self.end_headers() + self._log(404, inm, ims) + return + + st = os.stat(sPath) + nMod = int(st.st_mtime) + sETag = '"%x-%x"' % (int(st.st_mtime * 1000), st.st_size) + sLastMod = email.utils.formatdate(nMod, usegmt=True) + + if g_validators: + bCurrent = False + if inm is not None: + bCurrent = (inm == sETag) + elif ims is not None: + try: + bCurrent = email.utils.parsedate_to_datetime(ims).timestamp() >= nMod + except (TypeError, ValueError): + bCurrent = False + if bCurrent: + self.send_response(304) + self.send_header("ETag", sETag) + self.send_header("Last-Modified", sLastMod) + self.end_headers() + self._log(304, inm, ims) + return + + with open(sPath, "rb") as f: + xBody = f.read() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(xBody))) + if g_validators: + self.send_header("ETag", sETag) + self.send_header("Last-Modified", sLastMod) + self.end_headers() + self.wfile.write(xBody) + self._log(200, inm, ims) + + +def main(): + global g_dir, g_log, g_validators + psr = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + psr.add_argument("--dir", required=True, help="directory to serve") + psr.add_argument("--log", required=True, help="request log, appended") + psr.add_argument("--port-file", required=True, help="the chosen port is written here") + psr.add_argument("--no-validators", action="store_true", help="send no ETag or Last-Modified") + opts = psr.parse_args() + + g_dir = opts.dir + g_log = opts.log + g_validators = not opts.no_validators + + srv = HTTPServer(("127.0.0.1", 0), Handler) + with open(opts.port_file, "w") as f: + f.write("%d\n" % srv.server_port) + try: + srv.serve_forever() + except KeyboardInterrupt: + pass + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/das_cache_test.sh b/test/das_cache_test.sh new file mode 100755 index 0000000..c9084a6 --- /dev/null +++ b/test/das_cache_test.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash + +# Catalog node cache: the second run makes no request, an aged copy is +# revalidated with If-None-Match and renewed by a 304, a changed node is +# fetched again, an unreachable server leaves the old copy in use, and a +# server without validators still works, just without the saving. +# +# Runs against test/das_cache_srv.py on a loopback port picked at random, +# so nothing leaves the machine. The fixtures are copied to a scratch +# directory so one of them can be touched. Skips, passing, when python3 +# is not available. +# +# Drafted by Claude Fable 5.1, filed by Chris Piker. + +BD=$1 +FETCH="$BD/das_fetch" +FX=test/das_fetch + +if ! command -v python3 > /dev/null 2>&1; then + echo " SKIP: python3 not found, cache revalidation test not run" + exit 0 +fi + +nFail=0 +nPass=0 +check() { # check DESCRIPTION CONDITION... + local sWhat=$1; shift + if "$@"; then nPass=$((nPass + 1)); else echo " FAIL: $sWhat"; nFail=$((nFail + 1)); fi +} + +TMPD=$(mktemp -d) +SRVPID="" +cleanup() { [ -n "$SRVPID" ] && kill "$SRVPID" 2>/dev/null; rm -rf "$TMPD"; } +trap cleanup EXIT + +SRV=$TMPD/srv; mkdir -p "$SRV"; cp $FX/geomag_flex_0.7.json "$SRV/" +LOG=$TMPD/requests.log; : > "$LOG" +export DAS_USERDIR=$TMPD/userdir +CACHE=$DAS_USERDIR/cache/nodes + +start_server() { # start_server [--no-validators] + rm -f "$TMPD/port" + python3 test/das_cache_srv.py --dir "$SRV" --log "$LOG" --port-file "$TMPD/port" "$@" \ + 2> "$TMPD/srv.err" & + SRVPID=$! + # A cold python3 on a CI runner, macOS in particular, can take well over + # five seconds to get to the bind, so the wait is generous; a server + # that dies instead says why in srv.err + n=0 + while [ $n -lt 120 ] && [ ! -s "$TMPD/port" ]; do + kill -0 "$SRVPID" 2>/dev/null || break + sleep 0.5; n=$((n + 1)) + done + if [ ! -s "$TMPD/port" ]; then + echo " FAIL: test server did not start" + [ -s "$TMPD/srv.err" ] && sed 's/^/ srv: /' "$TMPD/srv.err" + exit 1 + fi + PORT=$(cat "$TMPD/port") +} +stop_server() { kill "$SRVPID" 2>/dev/null; wait "$SRVPID" 2>/dev/null; SRVPID=""; } + +mtime() { python3 -c "import os,sys; print(int(os.stat(sys.argv[1]).st_mtime))" "$1"; } + +EXPECT="http://127.0.0.1:8765/data/geomag.d2s?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" + +start_server +U="http://127.0.0.1:$PORT/geomag_flex_0.7.json" + +# 1. First contact: a plain GET, a 200, a cached body and its validators +sOut=$($FETCH url $U 0 2>/dev/null) +check "first run resolves the node" [ "$sOut" == "$EXPECT" ] +check "first run made exactly one request" [ "$(wc -l < "$LOG")" -eq 1 ] +check "first request was unconditional" grep -q "200 inm=- ims=-" "$LOG" +BODY=$(ls "$CACHE"/*.json 2>/dev/null | head -1) +check "a cached copy was written" [ -n "$BODY" ] && [ -s "$BODY" ] +check "its validators were saved beside it" grep -q "^ETag: " "$BODY.hdr" +check "Last-Modified saved too" grep -q "^Last-Modified: " "$BODY.hdr" + +# 2. Inside the trust window nothing is sent at all +sOut=$($FETCH url $U 0 2>/dev/null) +check "second run resolves from the cache" [ "$sOut" == "$EXPECT" ] +check "second run made no request" [ "$(wc -l < "$LOG")" -eq 1 ] + +# 3. -f forces a check: conditional GET, 304, body not resent, copy renewed +sleep 1.1 +nBefore=$(mtime "$BODY") +sOut=$($FETCH -f url $U 0 2>/dev/null) +check "forced check still resolves" [ "$sOut" == "$EXPECT" ] +check "forced check made one request" [ "$(wc -l < "$LOG")" -eq 2 ] +sETag=$(sed -n 's/^ETag: //p' "$BODY.hdr") +check "it carried the saved ETag" grep -q "304 inm=$sETag ims=" "$LOG" +check "it carried the saved date too" grep -qv "ims=-" <(tail -1 "$LOG") +check "the copy's age was renewed" [ "$(mtime "$BODY")" -gt "$nBefore" ] + +# 4. The node changes on the server: the next check downloads it again +sleep 1.1 +touch "$SRV/geomag_flex_0.7.json" +sOut=$($FETCH -f url $U 0 2>/dev/null) +check "changed node still resolves" [ "$sOut" == "$EXPECT" ] +check "changed node came back as a 200" grep -q "^GET .* 200 inm=$sETag" <(tail -1 "$LOG") +sNewETag=$(sed -n 's/^ETag: //p' "$BODY.hdr") +check "new validators replaced the old" [ -n "$sNewETag" ] && [ "$sNewETag" != "$sETag" ] + +# 5. Server gone: a forced check fails over to the cached copy with a warning +stop_server +sErr=$($FETCH -f url $U 0 2>&1 >/dev/null); nRet=$? +sOut=$($FETCH -f url $U 0 2>/dev/null) +check "unreachable server, cached copy used" [ "$sOut" == "$EXPECT" ] +check "and the run succeeds" [ $nRet -eq 0 ] +check "with a warning that says so" grep -q "cached copy" <<< "$sErr" + +# 6. A server with no validators: every forced check is a full fetch, no side file +rm -rf "$DAS_USERDIR"; : > "$LOG" +start_server --no-validators +U="http://127.0.0.1:$PORT/geomag_flex_0.7.json" +sOut=$($FETCH url $U 0 2>/dev/null) +check "plain server resolves" [ "$sOut" == "$EXPECT" ] +BODY=$(ls "$CACHE"/*.json 2>/dev/null | head -1) +check "plain server copy cached" [ -n "$BODY" ] +check "but no validators to save" [ ! -e "$BODY.hdr" ] +sOut=$($FETCH -f url $U 0 2>/dev/null) +check "forced check is unconditional" grep -q "200 inm=- ims=-" <(tail -1 "$LOG") +check "two full fetches in all" [ "$(grep -c " 200 " "$LOG")" -eq 2 ] +stop_server + +echo +if [ $nFail -eq 0 ]; then + echo " Result: PASSED ($nPass checks)" + exit 0 +else + echo " Result: FAILED ($nFail of $((nPass + nFail)) checks)" + exit 1 +fi diff --git a/test/das_fetch/cat/geomag.json b/test/das_fetch/cat/geomag.json new file mode 100644 index 0000000..6e129fe --- /dev/null +++ b/test/das_fetch/cat/geomag.json @@ -0,0 +1,14 @@ +{ + "type": "SourceSet", + "version": "0.5", + "name": "geomag", + "title": "Geomagnetic coordinates, offline test copy", + "separator": "/", + "catalog": { + "flex": { + "type": "HttpStreamSrc", + "title": "dasflex server", + "urls": ["test/das_fetch/geomag_flex_0.7.json"] + } + } +} diff --git a/test/das_fetch/cat/old.json b/test/das_fetch/cat/old.json new file mode 100644 index 0000000..1ac1ac6 --- /dev/null +++ b/test/das_fetch/cat/old.json @@ -0,0 +1,20 @@ +{ + "type": "Collection", + "version": "0.5", + "name": "old", + "title": "A 2018 layout container, entries say convention not provides", + "sources": { + "das2": { + "type": "HttpStreamSrc", + "convention": "das2/2.2", + "name": "Das2/2.2 Source", + "urls": ["test/das_fetch/juno_survey_0.6.json"] + }, + "hapi": { + "type": "HttpStreamSrc", + "convention": "hapi/2.0", + "name": "HAPI/2.0 Source", + "urls": ["test/das_fetch/juno_survey_0.6.json"] + } + } +} diff --git a/test/das_fetch/cat/site.json b/test/das_fetch/cat/site.json new file mode 100644 index 0000000..8dc9f3b --- /dev/null +++ b/test/das_fetch/cat/site.json @@ -0,0 +1,14 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "site", + "title": "Offline test catalog, stands in for tag:das2.org,2012:site:", + "separator": ":/", + "catalog": { + "uiowa": { + "type": "Catalog", + "title": "University of Iowa", + "urls": ["test/das_fetch/cat/uiowa.json"] + } + } +} diff --git a/test/das_fetch/cat/twin.json b/test/das_fetch/cat/twin.json new file mode 100644 index 0000000..b410cb2 --- /dev/null +++ b/test/das_fetch/cat/twin.json @@ -0,0 +1,30 @@ +{ + "type": "SourceSet", + "version": "0.5", + "name": "twin", + "title": "Two interfaces for one dataset, offline test copy", + "separator": "/", + "catalog": { + "flex": { + "type": "HttpStreamSrc", + "label": "dasflex server", + "purpose": "primary-stream", + "provides": ["application/vnd.das2.das2stream", "text/csv", "application/x-cdf"], + "urls": ["test/das_fetch/geomag_flex_0.7.json"] + }, + "das2": { + "type": "Das2DSDF", + "label": "Das2 DSDF, a type this library does not load", + "purpose": "primary-stream", + "provides": ["application/vnd.das2.das2stream"], + "urls": ["test/das_fetch/does_not_exist.d2t"] + }, + "das3": { + "type": "HttpStreamSrc", + "label": "das3 only server", + "purpose": "primary-stream", + "provides": ["application/vnd.das.stream"], + "urls": ["test/das_fetch/hsk_flex_0.7.json"] + } + } +} diff --git a/test/das_fetch/cat/uiowa.json b/test/das_fetch/cat/uiowa.json new file mode 100644 index 0000000..105e28c --- /dev/null +++ b/test/das_fetch/cat/uiowa.json @@ -0,0 +1,24 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "uiowa", + "title": "University of Iowa, offline test copy", + "separator": "/", + "catalog": { + "geomag": { + "type": "SourceSet", + "title": "Geomagnetic coordinates", + "urls": ["test/das_fetch/cat/geomag.json"] + }, + "broken": { + "type": "Catalog", + "title": "Listed, but its file does not exist", + "urls": ["test/das_fetch/cat/missing.json"] + }, + "odd": { + "type": "WebSocSrc", + "title": "A node type this library does not load", + "urls": ["test/das_fetch/cat/odd.json"] + } + } +} diff --git a/test/das_fetch/geomag_flex_0.7.json b/test/das_fetch/geomag_flex_0.7.json new file mode 100644 index 0000000..792f26f --- /dev/null +++ b/test/das_fetch/geomag_flex_0.7.json @@ -0,0 +1,383 @@ +{ + "label": "GeomagCoords", + "name": "ts2_geomag_coords", + "title": "TRACERS 2 Geomagnetic Location Coordinates, Predicted", + "type": "HttpStreamSrc", + "version": "0.7", + "protocol": { + "method": "GET", + "convention": "HTTP/1.1", + "baseUrls": [ + "http://127.0.0.1:8765/data/geomag.d2s", + "http://127.0.0.1:8765/data/geomag_alt.d2s" + ], + "authorization": { + "required": false + }, + "httpParams": { + "read.time.min": { + "required": true, + "type": "isotime" + }, + "read.time.max": { + "required": true, + "type": "isotime" + }, + "read.time.inter": { + "required": true, + "type": "real", + "units": "s" + }, + "read.data": { + "type": "FlagSet", + "required": false, + "flagSep": ",", + "flags": { + "lshell": { + "value": "lshell" + } + } + }, + "format.type": { + "required": false, + "type": "enum", + "enum": [ + "das", + "cdf", + "csv" + ] + }, + "format.secfrac": { + "required": false, + "type": "integer", + "range": [ + 0, + 9 + ] + }, + "format.sigdigit": { + "required": false, + "type": "integer", + "range": [ + 2, + 17 + ] + }, + "format.delim": { + "required": false, + "type": "string" + }, + "format.headers": { + "required": false, + "type": "boolean" + }, + "format.csv.props": { + "required": false, + "type": "boolean" + }, + "format.csv.dsid": { + "required": false, + "type": "boolean" + }, + "format.cdf.nocomp": { + "required": false, + "type": "boolean" + }, + "format.serial": { + "label": "Serialization", + "required": false, + "type": "enum", + "enum": [ + "text", + "binary" + ] + }, + "format.version": { + "required": false, + "type": "string" + } + } + }, + "contacts": [ + { + "type": "maintainer", + "name": "Kevin Steele", + "email": "kevin-steele@uiowa.edu" + }, + { + "type": "scientific", + "name": "Ivar Christopher" + } + ], + "interface": { + "coords": { + "time": { + "label": "Time", + "props": { + "min": { + "label": "Minimum", + "value": null, + "title": "Minimum time value to stream", + "set": { + "param": "read.time.min", + "required": true + } + }, + "max": { + "label": "Maximum", + "value": null, + "title": "Maximum time value to stream", + "set": { + "param": "read.time.max", + "required": true + } + }, + "units": { + "value": "UTC" + }, + "inter": { + "label": "Interval", + "value": null, + "units": "s", + "title": "The time interval between output points form the SPICE model", + "set": { + "param": "read.time.inter", + "required": false + } + } + }, + "validRange": [ + "2024-01-01T00:00", + "now" + ] + } + }, + "data": { + "lshell": { + "label": "L-Shell", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "lshell" + } + } + }, + "title": "L-Shell Index" + } + }, + "examples": [ + { + "settings": { + "coords/time/props/min": "2026-01-01", + "coords/time/props/max": "2026-01-02", + "coords/time/props/inter": "60" + }, + "label": "First day of 2026" + } + ], + "formats": { + "cdf": { + "label": "CDF file", + "title": "NASA Common Data Format", + "mimeTypes": [ + "application/x-cdf" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": false, + "set": { + "value": true, + "param": "format.type", + "pval": "cdf" + } + }, + "nocompress": { + "value": false, + "type": "boolean", + "label": "No-Compression", + "title": "Do not compress variables, needed for PDS compliant output.", + "set": { + "value": true, + "param": "format.cdf.nocomp", + "pval": "1" + } + } + } + }, + "das": { + "label": "das stream", + "title": "Streaming format for plots", + "mimeTypes": [ + "application/vnd.das2.das2stream", + "application/vnd.das.stream", + "text/vnd.das2.das2stream" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": true, + "set": { + "value": true, + "param": "format.type", + "pval": "das" + } + }, + "version": { + "label": "Stream Version", + "value": "das2", + "type": "enum" + }, + "serial": { + "label": "Serialization", + "value": "binary", + "set": { + "param": "format.serial", + "enum": [ + { + "value": "text" + }, + { + "value": "binary" + } + ] + } + }, + "fracSecs": { + "label": "Factional Seconds", + "title": "Number of fractional seconds digits in text streams", + "value": 3, + "set": { + "param": "format.secfrac" + } + }, + "sigDigits": { + "label": "Significant Digits", + "title": "General significant digits for values in text streams", + "value": 5, + "set": { + "param": "format.sigdigit" + } + } + }, + "order": [ + "enabled", + "version", + "serial", + "sigDigits", + "fracSecs" + ] + }, + "csv": { + "label": "CSV stream", + "title": "(Comma) Separated Values for export", + "mimeTypes": [ + "text/csv" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": false, + "set": { + "value": true, + "param": "format.type", + "pval": "csv" + } + }, + "attributes": { + "value": false, + "type": "boolean", + "label": "Attributes", + "title": "Output data object property rows", + "set": { + "value": true, + "param": "format.csv.props", + "pval": "1" + } + }, + "headers": { + "value": true, + "type": "boolean", + "label": "Headers", + "title": "Output column headers", + "set": { + "value": false, + "param": "format.headers", + "pval": "1" + } + }, + "datasetIds": { + "value": true, + "type": "boolean", + "label": "Dataset IDs", + "title": "Output the dataset ID column", + "pval": "1", + "set": { + "value": false, + "param": "format.csv.dsid", + "pval": "1" + } + }, + "delimiter": { + "value": "Comma", + "type": "enum", + "label": "Value Separator", + "set": { + "param": "format.delim", + "enum": [ + { + "value": "Comma", + "pval": "," + }, + { + "value": "Semicolon", + "pval": ";" + }, + { + "value": "Pipe", + "pval": "|" + }, + { + "value": "Tab", + "pval": "tab" + } + ] + } + }, + "fracSecs": { + "label": "Factional Seconds", + "title": "Number of fractional seconds digits in text streams", + "value": 3, + "set": { + "param": "format.secfrac" + } + }, + "sigDigits": { + "label": "Significant Digits", + "title": "General significant digits for values in text streams", + "value": 5, + "set": { + "param": "format.sigdigit" + } + } + }, + "order": [ + "enabled", + "attributes", + "headers", + "datasetIds", + "delimiter", + "fracSecs", + "sigDigits" + ] + } + } + } +} \ No newline at end of file diff --git a/test/das_fetch/hsk_flex_0.7.json b/test/das_fetch/hsk_flex_0.7.json new file mode 100644 index 0000000..ab3b4f5 --- /dev/null +++ b/test/das_fetch/hsk_flex_0.7.json @@ -0,0 +1,1706 @@ +{ + "label": "HSK", + "name": "TS2_CDPU_HSK", + "title": "TRACERS-2 CDPU: CDPU Internal State Housekeeping Level 1A", + "type": "HttpStreamSrc", + "version": "0.7", + "protocol": { + "method": "GET", + "convention": "HTTP/1.1", + "baseUrls": [ + "https://space.physics.uiowa.edu/das/stream/source/tracers/tracers-2/l1a/cdpu/hsk/flex" + ], + "authorization": { + "required": true, + "methods": [ + "http-basic" + ] + }, + "httpParams": { + "read.time.min": { + "required": true, + "type": "isotime" + }, + "read.time.max": { + "required": true, + "type": "isotime" + }, + "bin.time.max": { + "required": false, + "type": "real", + "units": "s" + }, + "read.data": { + "type": "FlagSet", + "required": false, + "flagSep": ",", + "flags": { + "time": { + "value": "time" + }, + "nmi_errors": { + "value": "nmi_errors" + }, + "irq3_errors": { + "value": "irq3_errors" + }, + "irq4_errors": { + "value": "irq4_errors" + }, + "irq6_errors": { + "value": "irq6_errors" + }, + "irq7_errors": { + "value": "irq7_errors" + }, + "irq8_errors": { + "value": "irq8_errors" + }, + "irqc_errors": { + "value": "irqc_errors" + }, + "nmi_total": { + "value": "nmi_total" + }, + "efi_task_errors": { + "value": "efi_task_errors" + }, + "lvps_task_errors": { + "value": "lvps_task_errors" + }, + "pps_task_errors": { + "value": "pps_task_errors" + }, + "sci_task_errors": { + "value": "sci_task_errors" + }, + "hsk_task_errors": { + "value": "hsk_task_errors" + }, + "memtest_errors": { + "value": "memtest_errors" + }, + "memscrub_errors": { + "value": "memscrub_errors" + }, + "cmd_fifo_errors": { + "value": "cmd_fifo_errors" + }, + "sci_fifo_errors": { + "value": "sci_fifo_errors" + }, + "hsk_fifo_errors": { + "value": "hsk_fifo_errors" + }, + "magic_debug": { + "value": "magic_debug" + }, + "mag_debug": { + "value": "mag_debug" + }, + "msc_debug": { + "value": "msc_debug" + }, + "efi_debug": { + "value": "efi_debug" + }, + "aci_debug": { + "value": "aci_debug" + }, + "ace_debug": { + "value": "ace_debug" + }, + "overc_efi_3p3": { + "value": "overc_efi_3p3" + }, + "overc_efi_p5": { + "value": "overc_efi_p5" + }, + "overc_efi_n5": { + "value": "overc_efi_n5" + }, + "overc_efi_p12": { + "value": "overc_efi_p12" + }, + "overc_efi_n12": { + "value": "overc_efi_n12" + }, + "overc_msc_p12": { + "value": "overc_msc_p12" + }, + "overc_msc_n12": { + "value": "overc_msc_n12" + }, + "overc_mag_p5": { + "value": "overc_mag_p5" + }, + "overc_mag_n5": { + "value": "overc_mag_n5" + }, + "overc_mag_p12": { + "value": "overc_mag_p12" + }, + "overc_mag_n12": { + "value": "overc_mag_n12" + }, + "overc_cdpu_3p3": { + "value": "overc_cdpu_3p3" + }, + "overv_lvps_3p3": { + "value": "overv_lvps_3p3" + }, + "overv_lvps_p5": { + "value": "overv_lvps_p5" + }, + "overv_lvps_n5": { + "value": "overv_lvps_n5" + }, + "overv_lvps_p12": { + "value": "overv_lvps_p12" + }, + "overv_lvps_n12": { + "value": "overv_lvps_n12" + }, + "overt_ace": { + "value": "overt_ace" + }, + "overv_ace_p5": { + "value": "overv_ace_p5" + }, + "underv_ace_p5": { + "value": "underv_ace_p5" + }, + "overv_ace_3p3": { + "value": "overv_ace_3p3" + }, + "underv_ace_3p3": { + "value": "underv_ace_3p3" + }, + "overv_ace_mcp": { + "value": "overv_ace_mcp" + }, + "overc_ace_mcp": { + "value": "overc_ace_mcp" + }, + "overv_ace_stack": { + "value": "overv_ace_stack" + }, + "overc_ace_stack": { + "value": "overc_ace_stack" + }, + "overt_aci": { + "value": "overt_aci" + }, + "overv_aci_p5": { + "value": "overv_aci_p5" + }, + "underv_aci_n5": { + "value": "underv_aci_n5" + }, + "overv_aci_3p3": { + "value": "overv_aci_3p3" + }, + "overv_aci_1p5": { + "value": "overv_aci_1p5" + }, + "overv_aci_mcp": { + "value": "overv_aci_mcp" + }, + "overc_aci_mcp": { + "value": "overc_aci_mcp" + }, + "overv_aci_esa": { + "value": "overv_aci_esa" + }, + "overv_aci_bulk": { + "value": "overv_aci_bulk" + }, + "overc_aci_esa": { + "value": "overc_aci_esa" + }, + "over_aci_count": { + "value": "over_aci_count" + }, + "edac_write_back": { + "value": "edac_write_back" + }, + "edac_mram_enable": { + "value": "edac_mram_enable" + }, + "edac_sram_enable": { + "value": "edac_sram_enable" + }, + "overt_lvps_cnvrt": { + "value": "overt_lvps_cnvrt" + }, + "overt_lvps_chass": { + "value": "overt_lvps_chass" + }, + "overt_lvps_vref": { + "value": "overt_lvps_vref" + }, + "roi_timer": { + "value": "roi_timer" + }, + "edac_mram_errors": { + "value": "edac_mram_errors" + }, + "edac_sram_errors": { + "value": "edac_sram_errors" + }, + "sram_corrected": { + "value": "sram_corrected" + }, + "sram_uncorrected": { + "value": "sram_uncorrected" + } + } + }, + "read.apid": { + "type": "enum", + "required": false, + "enum": [ + "x140" + ] + }, + "format.type": { + "required": false, + "type": "enum", + "enum": [ + "das", + "ccsds" + ] + }, + "format.secfrac": { + "required": false, + "type": "integer", + "range": [ + 0, + 9 + ] + }, + "format.sigdigit": { + "required": false, + "type": "integer", + "range": [ + 2, + 17 + ] + }, + "format.delim": { + "required": false, + "type": "string" + }, + "format.headers": { + "required": false, + "type": "boolean" + }, + "format.csv.props": { + "required": false, + "type": "boolean" + }, + "format.csv.dsid": { + "required": false, + "type": "boolean" + }, + "format.cdf.nocomp": { + "required": false, + "type": "boolean" + }, + "format.serial": { + "label": "Serialization", + "required": false, + "type": "enum", + "enum": [ + "text", + "binary" + ] + }, + "format.version": { + "required": false, + "type": "string" + } + } + }, + "contacts": [ + { + "type": "maintainer", + "name": "Chris Piker", + "email": "chris-piker@uiowa.edu" + }, + { + "type": "scientific", + "name": "Mark Chutter" + } + ], + "interface": { + "coords": { + "time": { + "label": "Time", + "props": { + "min": { + "label": "Minimum", + "value": null, + "title": "Minimum time value to stream", + "set": { + "param": "read.time.min", + "required": true + } + }, + "max": { + "label": "Maximum", + "value": null, + "title": "Maximum time value to stream", + "set": { + "param": "read.time.max", + "required": true + } + }, + "units": { + "value": "UTC" + }, + "res": { + "label": "Resolution", + "value": null, + "units": "s", + "title": "The maximum time bin width for bin-reduced data in seconds", + "set": { + "param": "bin.time.max", + "required": false + } + } + }, + "validRange": [ + "2025-08-04T22:49", + "2026-01-06T14:24" + ] + } + }, + "examples": [ + { + "settings": { + "coords/time/props/min": "2026-01-06T14:12", + "coords/time/props/max": "2026-01-06T14:24" + }, + "label": "Most Recent 10 minutes (das3)" + }, + { + "settings": { + "coords/time/props/min": "2026-01-05T14:24", + "coords/time/props/max": "2026-01-06T14:24", + "formats/cdf/props/enabled": true + }, + "label": "Most Recent Day (CDF)" + } + ], + "formats": { + "cdf": { + "label": "CDF file", + "title": "NASA Common Data Format", + "mimeTypes": [ + "application/x-cdf" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": false, + "set": { + "value": true, + "param": "format.type", + "pval": "cdf" + } + }, + "nocompress": { + "value": false, + "type": "boolean", + "label": "No-Compression", + "title": "Do not compress variables, needed for PDS compliant output.", + "set": { + "value": true, + "param": "format.cdf.nocomp", + "pval": "1" + } + } + } + }, + "das": { + "label": "das stream", + "title": "Streaming format for plots", + "mimeTypes": [ + "application/vnd.das2.das2stream", + "application/vnd.das.stream", + "text/vnd.das2.das2stream" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": true, + "set": { + "value": true, + "param": "format.type", + "pval": "das" + } + }, + "version": { + "label": "Stream Version", + "value": "das3", + "type": "enum", + "set": { + "param": "format.version", + "enum": [ + { + "value": "das3", + "pval": "3" + }, + { + "value": "das2", + "pval": "2" + } + ] + } + }, + "serial": { + "label": "Serialization", + "value": "binary", + "set": { + "param": "format.serial", + "enum": [ + { + "value": "text" + }, + { + "value": "binary" + } + ] + } + }, + "fracSecs": { + "label": "Factional Seconds", + "title": "Number of fractional seconds digits in text streams", + "value": 3, + "set": { + "param": "format.secfrac" + } + }, + "sigDigits": { + "label": "Significant Digits", + "title": "General significant digits for values in text streams", + "value": 5, + "set": { + "param": "format.sigdigit" + } + } + }, + "order": [ + "enabled", + "version", + "serial", + "sigDigits", + "fracSecs" + ] + }, + "csv": { + "label": "CSV stream", + "title": "(Comma) Separated Values for export", + "mimeTypes": [ + "text/csv" + ], + "props": { + "enabled": { + "type": "boolean", + "xorGroup": "format", + "value": false, + "set": { + "value": true, + "param": "format.type", + "pval": "csv" + } + }, + "attributes": { + "value": false, + "type": "boolean", + "label": "Attributes", + "title": "Output data object property rows", + "set": { + "value": true, + "param": "format.csv.props", + "pval": "1" + } + }, + "headers": { + "value": true, + "type": "boolean", + "label": "Headers", + "title": "Output column headers", + "set": { + "value": false, + "param": "format.headers", + "pval": "1" + } + }, + "datasetIds": { + "value": true, + "type": "boolean", + "label": "Dataset IDs", + "title": "Output the dataset ID column", + "pval": "1", + "set": { + "value": false, + "param": "format.csv.dsid", + "pval": "1" + } + }, + "delimiter": { + "value": "Comma", + "type": "enum", + "label": "Value Separator", + "set": { + "param": "format.delim", + "enum": [ + { + "value": "Comma", + "pval": "," + }, + { + "value": "Semicolon", + "pval": ";" + }, + { + "value": "Pipe", + "pval": "|" + }, + { + "value": "Tab", + "pval": "tab" + } + ] + } + }, + "fracSecs": { + "label": "Factional Seconds", + "title": "Number of fractional seconds digits in text streams", + "value": 3, + "set": { + "param": "format.secfrac" + } + }, + "sigDigits": { + "label": "Significant Digits", + "title": "General significant digits for values in text streams", + "value": 5, + "set": { + "param": "format.sigdigit" + } + } + }, + "order": [ + "enabled", + "attributes", + "headers", + "datasetIds", + "delimiter", + "fracSecs", + "sigDigits" + ] + } + }, + "data": { + "time": { + "label": "time", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "time" + } + } + } + }, + "nmi_errors": { + "label": "NMI Errs", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "nmi_errors" + } + } + }, + "title": "NMI Error Counter" + }, + "irq3_errors": { + "label": "Cnt IRQ3 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq3_errors" + } + } + }, + "title": "IRQ-3 error counter" + }, + "irq4_errors": { + "label": "Cnt IRQ4 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq4_errors" + } + } + }, + "title": "IRQ-4 error counter" + }, + "irq6_errors": { + "label": "Cnt IRQ6 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq6_errors" + } + } + }, + "title": "IRQ-6 error counter" + }, + "irq7_errors": { + "label": "Cnt IRQ7 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq7_errors" + } + } + }, + "title": "IRQ-7 error counter" + }, + "irq8_errors": { + "label": "Cnt IRQ8 Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irq8_errors" + } + } + }, + "title": "IRQ-8 error counter" + }, + "irqc_errors": { + "label": "Cnt IRQC Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "irqc_errors" + } + } + }, + "title": "IRQ-C error counter" + }, + "nmi_total": { + "label": "Count NMI", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "nmi_total" + } + } + }, + "title": "count_nmi warning counter" + }, + "efi_task_errors": { + "label": "EFI Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "efi_task_errors" + } + } + }, + "title": "efi_task_error error counter" + }, + "lvps_task_errors": { + "label": "LVPS Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "lvps_task_errors" + } + } + }, + "title": "lvps_task_error error counter" + }, + "pps_task_errors": { + "label": "IRQD Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "pps_task_errors" + } + } + }, + "title": "irqd_task_error error counter" + }, + "sci_task_errors": { + "label": "Sci Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "sci_task_errors" + } + } + }, + "title": "sci_task_error error counter" + }, + "hsk_task_errors": { + "label": "HK Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "hsk_task_errors" + } + } + }, + "title": "hk_task_error error counter" + }, + "memtest_errors": { + "label": "Test Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "memtest_errors" + } + } + }, + "title": "test_task_error error counter" + }, + "memscrub_errors": { + "label": "Scrb Task Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "memscrub_errors" + } + } + }, + "title": "scrub_task_error error counter" + }, + "cmd_fifo_errors": { + "label": "TC FIFO Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "cmd_fifo_errors" + } + } + }, + "title": "tc_fifo_error error counter" + }, + "sci_fifo_errors": { + "label": "SC FIFO Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "sci_fifo_errors" + } + } + }, + "title": "sc_fifo_error error counter" + }, + "hsk_fifo_errors": { + "label": "HK FIFO Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "hsk_fifo_errors" + } + } + }, + "title": "hk_fifo_error error counter" + }, + "magic_debug": { + "label": "MAGIC Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "magic_debug" + } + } + }, + "title": "magic_tm debug flag" + }, + "mag_debug": { + "label": "MAG Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "mag_debug" + } + } + }, + "title": "mag_tm debug flag" + }, + "msc_debug": { + "label": "MSC Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "msc_debug" + } + } + }, + "title": "msc_tm debug flag" + }, + "efi_debug": { + "label": "EFI Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "efi_debug" + } + } + }, + "title": "efi_tm debug flag" + }, + "aci_debug": { + "label": "ACI Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "aci_debug" + } + } + }, + "title": "aci_tm debug flag" + }, + "ace_debug": { + "label": "ACE Debug", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "ace_debug" + } + } + }, + "title": "ace_tm debug flag" + }, + "overc_efi_3p3": { + "label": "EFI 3.3V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_3p3" + } + } + }, + "title": "Over-current flag for EFI 3.3V" + }, + "overc_efi_p5": { + "label": "EFI +5V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_p5" + } + } + }, + "title": "Over-current flag for EFI +5V" + }, + "overc_efi_n5": { + "label": "EFI -5V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_n5" + } + } + }, + "title": "Over-current flag for EFI -5V" + }, + "overc_efi_p12": { + "label": "EFI +12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_p12" + } + } + }, + "title": "Over-current flag for EFI +12V" + }, + "overc_efi_n12": { + "label": "EFI -12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_efi_n12" + } + } + }, + "title": "Over-current flag for EFI -12V" + }, + "overc_msc_p12": { + "label": "MSC +12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_msc_p12" + } + } + }, + "title": "Over-current flag for MSC +12V" + }, + "overc_msc_n12": { + "label": "MSC -12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_msc_n12" + } + } + }, + "title": "Over-current flag for MSC -12V" + }, + "overc_mag_p5": { + "label": "MAG +5V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_mag_p5" + } + } + }, + "title": "Over-current flag for MAG +5V" + }, + "overc_mag_n5": { + "label": "MAG -5V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_mag_n5" + } + } + }, + "title": "Over-current flag for MAG -5V" + }, + "overc_mag_p12": { + "label": "MAG +12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_mag_p12" + } + } + }, + "title": "Over-current flag for MAG +12V" + }, + "overc_mag_n12": { + "label": "MAG -12V OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_mag_n12" + } + } + }, + "title": "Over-current flag for MAG -12V" + }, + "overc_cdpu_3p3": { + "label": "CDPU OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_cdpu_3p3" + } + } + }, + "title": "Over-current flag for CDPU" + }, + "overv_lvps_3p3": { + "label": "LVPS 3.3V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_3p3" + } + } + }, + "title": "Over-voltage flag for LVPS 3.3V" + }, + "overv_lvps_p5": { + "label": "LVPS +5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_p5" + } + } + }, + "title": "Over-voltage flag for LVPS +5V" + }, + "overv_lvps_n5": { + "label": "LVPS -5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_n5" + } + } + }, + "title": "Over-voltage flag for LVPS -5V" + }, + "overv_lvps_p12": { + "label": "LVPS +12V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_p12" + } + } + }, + "title": "Over-voltage flag for LVPS +12V" + }, + "overv_lvps_n12": { + "label": "LVPS -12V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_lvps_n12" + } + } + }, + "title": "Over-voltage flag for LVPS -12V" + }, + "overt_ace": { + "label": "ACE OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_ace" + } + } + }, + "title": "Over-temp flag for ACE" + }, + "overv_ace_p5": { + "label": "ACE +5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_ace_p5" + } + } + }, + "title": "Over-voltage flag for ACE +5V" + }, + "underv_ace_p5": { + "label": "ACE +5V UV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "underv_ace_p5" + } + } + }, + "title": "Under-voltage flag for ACE +5V" + }, + "overv_ace_3p3": { + "label": "ACE 3.3V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_ace_3p3" + } + } + }, + "title": "Over-voltage flag for ACE 3.3V" + }, + "underv_ace_3p3": { + "label": "ACE 3.3V UV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "underv_ace_3p3" + } + } + }, + "title": "Under-voltage flag for ACE 3.3V" + }, + "overv_ace_mcp": { + "label": "ACE MCP OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_ace_mcp" + } + } + }, + "title": "Over-voltage flag for ACE MCP" + }, + "overc_ace_mcp": { + "label": "ACE MCP OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_ace_mcp" + } + } + }, + "title": "Over-current flag for ACE MCP" + }, + "overv_ace_stack": { + "label": "ACE Stack OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_ace_stack" + } + } + }, + "title": "Over-voltage flag for ACE Stack" + }, + "overc_ace_stack": { + "label": "ACE Stack OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_ace_stack" + } + } + }, + "title": "Over-current flag for ACE Stack" + }, + "overt_aci": { + "label": "ACI OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_aci" + } + } + }, + "title": "Over-temp flag for ACI" + }, + "overv_aci_p5": { + "label": "ACI +5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_p5" + } + } + }, + "title": "Over-voltage flag for ACI +5V" + }, + "underv_aci_n5": { + "label": "ACI -5V UV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "underv_aci_n5" + } + } + }, + "title": "Under-voltage flag for ACI -5V" + }, + "overv_aci_3p3": { + "label": "ACI 3.3V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_3p3" + } + } + }, + "title": "Over-voltage flag for ACI 3.3V" + }, + "overv_aci_1p5": { + "label": "ACI 1.5V OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_1p5" + } + } + }, + "title": "Over-voltage flag for ACI 1.5V" + }, + "overv_aci_mcp": { + "label": "ACI MCP OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_mcp" + } + } + }, + "title": "Over-voltage flag for ACI MCP" + }, + "overc_aci_mcp": { + "label": "ACI MCP OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_aci_mcp" + } + } + }, + "title": "Over-current flag for ACI MCP" + }, + "overv_aci_esa": { + "label": "ACI ESA OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_esa" + } + } + }, + "title": "Over-voltage flag for ACI ESA" + }, + "overv_aci_bulk": { + "label": "ACI Bulk OV", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overv_aci_bulk" + } + } + }, + "title": "Over-voltage flag for ACI Bulk" + }, + "overc_aci_esa": { + "label": "ACI ESA OC", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overc_aci_esa" + } + } + }, + "title": "Over-current flag for ACI ESA" + }, + "over_aci_count": { + "label": "ACI OverCnt", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "over_aci_count" + } + } + }, + "title": "Over-count flag for ACI" + }, + "edac_write_back": { + "label": "EDAC WrtBk", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_write_back" + } + } + }, + "title": "EDAC write back active" + }, + "edac_mram_enable": { + "label": "EDAC MRAM", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_mram_enable" + } + } + }, + "title": "MRAM EDAC checking active" + }, + "edac_sram_enable": { + "label": "EDAC SRAM", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_sram_enable" + } + } + }, + "title": "SRAM EDAC checking active" + }, + "overt_lvps_cnvrt": { + "label": "LVPS Cnv OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_lvps_cnvrt" + } + } + }, + "title": "Over-temp flag for LVPS converter" + }, + "overt_lvps_chass": { + "label": "LVPS Chs OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_lvps_chass" + } + } + }, + "title": "Over-temp flag for LVPS chassis" + }, + "overt_lvps_vref": { + "label": "LVPS Vref OT", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "overt_lvps_vref" + } + } + }, + "title": "Over-temp flag for LVPS vref" + }, + "roi_timer": { + "label": "ROI CntDn Tmr", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "roi_timer" + } + } + }, + "title": "ROI countdown timer" + }, + "edac_mram_errors": { + "label": "EDAC MRAM Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_mram_errors" + } + } + }, + "title": "Counter of EDAC MRAM errors" + }, + "edac_sram_errors": { + "label": "EDAC SRAM Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "edac_sram_errors" + } + } + }, + "title": "Counter of EDAC SRAM errors" + }, + "sram_corrected": { + "label": "SBit Cor Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "sram_corrected" + } + } + }, + "title": "Count of single-bit correctable errors" + }, + "sram_uncorrected": { + "label": "EDAC SRAM Err", + "props": { + "enabled": { + "type": "boolean", + "value": false, + "set": { + "value": true, + "param": "read.data", + "flag": "sram_uncorrected" + } + } + }, + "title": "Counter of EDAC SRAM errors" + } + } + } +} \ No newline at end of file diff --git a/test/das_fetch/juno_survey_0.6.json b/test/das_fetch/juno_survey_0.6.json new file mode 100644 index 0000000..be552b3 --- /dev/null +++ b/test/das_fetch/juno_survey_0.6.json @@ -0,0 +1,348 @@ +{ + "format": { + "default": { + "mime": "application/vnd.das2.das2stream", + "name": "Das2 Stream" + } + }, + "interface": { + "coordinates": { + "frequency": { + "name": "Frequency", + "title": "Y", + "units": { + "value": "Hz" + } + }, + "time": { + "maximum": { + "set": { + "param": "end_time", + "range": [ + "2010-09-01T00:00:00", + "now" + ], + "required": true + }, + "value": "2018-10-04T11:00" + }, + "minimum": { + "set": { + "param": "start_time", + "range": [ + "2010-09-01T00:00:00", + "now" + ], + "required": true + }, + "value": "2018-10-02T11:00" + }, + "name": "Time", + "resolution": { + "set": { + "param": "resolution", + "required": false + }, + "units": "s", + "value": 86.4 + }, + "units": { + "value": "UTC" + } + } + }, + "data": { + "specdens": { + "name": "Specdens", + "title": "Z", + "units": { + "value": "V**2 m**-2 Hz**-1" + } + } + }, + "options": { + "ascii": { + "set": { + "flag": "00", + "param": "params", + "value": true + }, + "title": "If specified, output is in UTF-8 format", + "type": "boolean", + "value": false + }, + "b": { + "set": { + "flag": "03", + "param": "params", + "value": true + }, + "title": "Only output magnetic data from the low-band of the Low Frequency Receiver", + "type": "boolean", + "value": false + }, + "das2_text": { + "name": "das2 text", + "set": { + "param": "ascii", + "pval": "true", + "value": true + }, + "title": "Convert stream to das2 text (utf-8) format", + "value": false + }, + "hfrh": { + "set": { + "flag": "07", + "param": "params", + "value": true + }, + "title": "Only output data from the frequency down-mixed sections of the High Frequency Receivers", + "type": "boolean", + "value": false + }, + "hfrl": { + "set": { + "flag": "06", + "param": "params", + "value": true + }, + "title": "Only output data from the baseband of the High Frequency Receivers", + "type": "boolean", + "value": false + }, + "keep-spikes": { + "set": { + "flag": "01", + "param": "params", + "value": true + }, + "title": "If specified single LFR packets with a pre-amp toggle are retained in the output", + "type": "boolean", + "value": false + }, + "lfrh": { + "set": { + "flag": "05", + "param": "params", + "value": true + }, + "title": "Only output data from the high-band of the Low Frequency Receiver", + "type": "boolean", + "value": false + }, + "lfrl": { + "set": { + "flag": "04", + "param": "params", + "value": true + }, + "title": "Only output electric from the low-band of the Low Frequency Receiver", + "type": "boolean", + "value": false + }, + "xmas": { + "set": { + "flag": "02", + "param": "params", + "value": true + }, + "title": "If specified only output the Xmas flag", + "type": "boolean", + "value": false + } + } + }, + "name": "Das2/2.2 Source", + "protocol": { + "authentication": { + "realm": "Juno Magnetospheric Working Group", + "required": true + }, + "base_urls": [ + "http://127.0.0.1:8765/data/juno_survey.d2s?server=dataset&dataset=Juno/WAV/Survey" + ], + "convention": "das2/2.2", + "examples": { + "example_00": { + "http_params": { + "end_time": "2018-10-04T11:00", + "resolution": 86.4, + "start_time": "2018-10-02T11:00" + }, + "name": "Example 00", + "title": "Perijove 16", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-10-04T11%3A00&start_time=2018-10-02T11%3A00" + }, + "example_01": { + "http_params": { + "end_time": "2018-08-12T15:00", + "resolution": 86.4, + "start_time": "2018-08-10T15:00" + }, + "name": "Example 01", + "title": "Perijove 15", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-08-12T15%3A00&start_time=2018-08-10T15%3A00" + }, + "example_02": { + "http_params": { + "end_time": "2018-06-20T18:00", + "resolution": 86.4, + "start_time": "2018-06-18T18:00" + }, + "name": "Example 02", + "title": "Perijove 14", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-06-20T18%3A00&start_time=2018-06-18T18%3A00" + }, + "example_03": { + "http_params": { + "end_time": "2018-04-28T20:00", + "resolution": 86.4, + "start_time": "2018-04-26T20:00" + }, + "name": "Example 03", + "title": "Perijove 13", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-04-28T20%3A00&start_time=2018-04-26T20%3A00" + }, + "example_04": { + "http_params": { + "end_time": "2018-03-07T00:00", + "resolution": 86.4, + "start_time": "2018-03-05T00:00" + }, + "name": "Example 04", + "title": "Perijove 12", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-03-07T00%3A00&start_time=2018-03-05T00%3A00" + }, + "example_05": { + "http_params": { + "end_time": "2018-01-13T04:00", + "resolution": 86.4, + "start_time": "2018-01-11T04:00" + }, + "name": "Example 05", + "title": "Perijove 11", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=86.4&end_time=2018-01-13T04%3A00&start_time=2018-01-11T04%3A00" + }, + "example_15": { + "http_params": { + "end_time": "2016-177T00:00", + "resolution": 43.2, + "start_time": "2016-176T00:00" + }, + "name": "Example 15", + "title": "Jupiter Bow Shock", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=43.2&end_time=2016-177T00%3A00&start_time=2016-176T00%3A00" + }, + "example_16": { + "http_params": { + "end_time": "2016-177T00:00", + "params": "B", + "resolution": 43.2, + "start_time": "2016-176T00:00" + }, + "name": "Example 16", + "title": "Jupiter Bow Shock, Magnetic", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=43.2&end_time=2016-177T00%3A00¶ms=B&start_time=2016-176T00%3A00" + }, + "example_98": { + "http_params": { + "end_time": "2013-10-10", + "resolution": 43.2, + "start_time": "2013-10-09" + }, + "name": "Example 98", + "title": "Earth Flyby", + "url": "https://jupiter.physics.uiowa.edu/das/server?server=dataset&dataset=Juno/WAV/Survey&resolution=43.2&end_time=2013-10-10&start_time=2013-10-09" + } + }, + "http_params": { + "ascii": { + "name": "UTF-8", + "required": false, + "title": "Insure stream output is readable as UTF-8 text", + "type": "boolean" + }, + "end_time": { + "name": "Max Time", + "required": true, + "title": "Maximum Time Value to stream", + "type": "isotime" + }, + "params": { + "flag_sep": " ", + "flags": { + "00": { + "name": "ASCII", + "title": "If specified, output is in UTF-8 format", + "value": "ASCII" + }, + "01": { + "name": "KEEP-SPIKES", + "title": "If specified single LFR packets with a pre-amp toggle are retained in the output", + "value": "KEEP-SPIKES" + }, + "02": { + "name": "XMAS", + "title": "If specified only output the Xmas flag", + "value": "XMAS" + }, + "03": { + "name": "B", + "title": "Only output magnetic data from the low-band of the Low Frequency Receiver", + "value": "B" + }, + "04": { + "name": "LFRL", + "title": "Only output electric from the low-band of the Low Frequency Receiver", + "value": "LFRL" + }, + "05": { + "name": "LFRH", + "title": "Only output data from the high-band of the Low Frequency Receiver", + "value": "LFRH" + }, + "06": { + "name": "HFRL", + "title": "Only output data from the baseband of the High Frequency Receivers", + "value": "HFRL" + }, + "07": { + "name": "HFRH", + "title": "Only output data from the frequency down-mixed sections of the High Frequency Receivers", + "value": "HFRH" + } + }, + "required": false, + "title": "Optional reader arguments", + "type": "flag_set" + }, + "resolution": { + "description": "The server will return data at or better than the given resolution if possible. Leave un-specified to get data at intrinsic resolution without server side averages", + "name": "Resolution", + "required": false, + "title": "Maximum resolution between output time points", + "type": "real", + "units": "s" + }, + "start_time": { + "name": "Min Time", + "required": true, + "title": "Minimum time value to stream", + "type": "isotime" + } + } + }, + "tech_contacts": [ + { + "email": "chris-piker@uiowa.edu", + "name": "Chris Piker" + } + ], + "title": "Survey Mode Electric or Magnetic Spectral Densities", + "type": "HttpStreamSrc", + "uris": [ + "tag:physics.uiowa.edu,2018:das.juno.wav.survey.das2" + ], + "version": "0.6" +} \ No newline at end of file diff --git a/test/das_fetch_test.sh b/test/das_fetch_test.sh new file mode 100755 index 0000000..cbdcd39 --- /dev/null +++ b/test/das_fetch_test.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash + +# Regression test for das_fetch's interface-to-protocol translation. Runs +# entirely offline: the fixtures in test/das_fetch/ are real catalog nodes +# saved from the uiowa servers with their base URLs pointed at localhost, +# and the 'url' command resolves a query without contacting anything. +# +# Three node generations are covered: +# geomag_flex_0.7.json dasflex HttpStreamSrc 0.7, required interval param, +# FlagSet data toggle, csv/cdf/das format groups +# hsk_flex_0.7.json 0.7 with 73 data variables and a das2/das3 version enum +# juno_survey_0.6.json 2018 era HttpStreamSrc 0.6, flag_set with space +# separator, base URL that already has a query string +# +# Each case is: arguments, then the exact URL expected on stdout. Error cases +# check the exit code instead. +# +# Drafted by Claude Fable 5.1, filed by Chris Piker. + +BD=$1 +FETCH="$BD/das_fetch -C none" # no node cache, the fixtures are files anyway +FX=test/das_fetch + +nFail=0 +nPass=0 + +# expect_url ARGS... -- EXPECTED +expect_url() { + local args=() + while [ "$1" != "--" ]; do args+=("$1"); shift; done + shift + local expected="$1" + local actual + actual=$($FETCH url "${args[@]}" 2>/dev/null) + if [ "$actual" == "$expected" ]; then + nPass=$((nPass + 1)) + else + echo " FAIL: das_fetch url ${args[*]}" + echo " expected: $expected" + echo " actual: $actual" + nFail=$((nFail + 1)) + fi +} + +# expect_rc CODE ARGS... +expect_rc() { + local want=$1; shift + $FETCH "$@" > /dev/null 2>&1 + local got=$? + if [ $got -eq $want ]; then + nPass=$((nPass + 1)) + else + echo " FAIL: das_fetch $* : expected exit $want, got $got" + nFail=$((nFail + 1)) + fi +} + +G=$FX/geomag_flex_0.7.json +H=$FX/hsk_flex_0.7.json +J=$FX/juno_survey_0.6.json +GB="http://127.0.0.1:8765/data/geomag.d2s" +HB="https://space.physics.uiowa.edu/das/stream/source/tracers/tracers-2/l1a/cdpu/hsk/flex" +JB="http://127.0.0.1:8765/data/juno_survey.d2s?server=dataset&dataset=Juno/WAV/Survey" + +# --- 0.7 geomag: ranges, shortcuts, aliases, flags, examples, formats -------- + +expect_url $G time=2026-04-01,2026-04-01T01:30,60 lshell -- \ + "$GB?read.time.min=2026-04-01&read.time.max=2026-04-01T01%3A30&read.time.inter=60&read.data=lshell" + +# minimum/maximum spellings work on a 0.7 node, section prefix optional +expect_url $G coord.time.minimum=2026-02-01 time.maximum=2026-02-02 inter=60 -- \ + "$GB?read.time.min=2026-02-01&read.time.max=2026-02-02&read.time.inter=60" + +# example 0, then an explicit key overrides one of its settings +expect_url $G 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url $G 0 time.inter=30 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30" + +# a boolean set to its default is not sent +expect_url $G 0 lshell=no -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" + +# format group by name, enum pval substitution (Tab -> tab), boolean with pval +expect_url $G 0 csv delimiter=Tab csv.headers=false serial=text -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60&format.serial=text&format.type=csv&format.headers=1&format.delim=tab" + +# example by bare number, by examples.N, and as the default when nothing is asked +expect_url $G 0 time.inter=30 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30" +expect_url $G examples.0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url $G -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" + +# -r names the seed node, URL=URI, and PATH is then optional. A bare +# location is a seed with no name, a root, which works too; the old +# URI=URL order is refused with a hint. +SU="tag:das2.org,2012:site:/uiowa/tracers/tracers-2/spice/geomagcoords/flex" +HU="tag:das2.org,2012:site:/uiowa/tracers/tracers-2/l1a/cdpu/hsk/flex" +expect_url -r $G 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" # a root seed +expect_rc 2 url -r =$G 0 # nothing before the '=' +expect_rc 2 url -r $SU=$G 0 # the old order, URI=URL +expect_url -r $G=$SU 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url -r $G=$SU $SU 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +if $FETCH node -r $G=$SU 2>/dev/null | grep -q "^URI: $SU\$"; then + nPass=$((nPass + 1)) +else + echo " FAIL: das_fetch node -r $G=$SU did not report the seed URI"; nFail=$((nFail + 1)) +fi + +# -m mounts the same node on the federated catalog. Offline only the +# mounted side can be exercised, and a mount is not a place to start, so +# the path is named; -m and -r URL together is a usage error. +expect_url -m $G=$SU $SU 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_rc 2 url -m $G=$SU 0 # a mount gives no default path +export DASFETCH_MOUNT=$G=$SU +expect_url $SU 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +unset DASFETCH_MOUNT +expect_url -m $G=$SU -r $H=$HU 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" # a seed and a mount make an offline catalog +expect_url -m $G=$SU -r $H=$HU $SU 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_rc 2 url -m site:/uiowa 0 # mount needs URL=URI +# Under a root seed a mount is named by a path from the root, and it is +# consulted for the relative spelling, shadowing the listing +expect_url -r $FX/cat/site.json -m $G=uiowa/broken uiowa/broken 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_rc 2 url -r nourl=site:/uiowa 0 # '=' but no URL before it +expect_rc 3 url -r $G=$SU site:/uiowa/juno # a seed is the whole catalog + +# DASFETCH_START stands in for -r, and -r still wins over it +export DASFETCH_START=$G=$SU +expect_url 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url -r $H=$HU 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" +unset DASFETCH_START + +# dotted member paths, section names may be plural or singular +expect_url $G 0 data.lshell coord.time.inter=30 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=30&read.data=lshell" + +# raw protocol parameters override translated ones and may add new ones +expect_url $G 0 -p read.time.inter=5 -p extra=1 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=5&extra=1" + +# --- 0.7 hsk: version enum, FlagSet accumulation with comma, cdf example ---- + +# the node default (das3) is not sent, only choices are +expect_url $H 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" +expect_url $H 0 version=das2 nmi_errors irq3_errors data.hsk_task_errors -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24&read.data=nmi_errors%2Cirq3_errors%2Chsk_task_errors&format.version=2" +expect_url $H 1 -- \ + "$HB?read.time.min=2026-01-05T14%3A24&read.time.max=2026-01-06T14%3A24&format.type=cdf" +expect_url $H 0 formats.csv das.version=das2 time.res=10 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24&bin.time.max=10&format.version=2&format.type=csv" + +# --- 0.6 juno: '&' joiner, space separated flags, protocol level examples --- + +expect_url $J time=2021-02-21,2021-02-22 b lfrl das2_text -- \ + "$JB&end_time=2021-02-22&start_time=2021-02-21¶ms=B%20LFRL&ascii=true" +expect_url $J time=2021-02-21,2021-02-22,0 ascii option.xmas=true -- \ + "$JB&end_time=2021-02-22&start_time=2021-02-21&resolution=0¶ms=ASCII%20XMAS" + +# a 0.6 example is raw parameters, read back into interface terms, so the +# URL comes out in interface order; user keys still win, required defaults +# must not clobber the example's values +expect_url $J 7 resolution=10 -- \ + "$JB&end_time=2016-177T00%3A00&start_time=2016-176T00%3A00&resolution=10¶ms=B" + +# --- choosing among a node's sources by -a --------------------------------- + +# twin.json offers flex (das2 stream, csv, cdf) and das3 (das3 stream); with +# no -a das_fetch asks for das3 streams, then anything. An -a list is a set. +T=$FX/cat/twin.json +O=$FX/cat/old.json +expect_url -r $T=site:/uiowa/twin 0 -- \ + "$HB?read.time.min=2026-01-06T14%3A12&read.time.max=2026-01-06T14%3A24" +expect_url -r $T=site:/uiowa/twin -a text/csv 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url -r $T=site:/uiowa/twin -a 'image/png, application/x-cdf' 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_rc 3 url -r $T=site:/uiowa/twin -a '*/*' 0 # two sources qualify, name one +expect_rc 3 url -r $T=site:/uiowa/twin -a 'application/vnd.das.stream, text/csv' 0 # order does not pick +expect_rc 3 url -r $T=site:/uiowa/twin -a image/png 0 # none does +expect_url -r $T=site:/uiowa/twin site:/uiowa/twin/flex -a '*/*' 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +if $FETCH node -r $T=site:/uiowa/twin -a '*/*' 2>/dev/null | grep -q "^ das3 "; then + nPass=$((nPass + 1)) +else + echo " FAIL: das_fetch node with an ambiguous pick did not list the children"; nFail=$((nFail + 1)) +fi + +# a 2018 Collection has no provides lists, convention stands in +expect_url -r $O=site:/uiowa/old -a application/vnd.das2.das2stream time=2021-02-21,2021-02-22 -- \ + "$JB&end_time=2021-02-22&start_time=2021-02-21" +expect_rc 3 url -r $O=site:/uiowa/old time=2021-02-21,2021-02-22 # nothing provides das3, so anything: both qualify +expect_rc 3 url -r $O=site:/uiowa/old -a text/csv time=2021-02-21,2021-02-22 + +# -m may be repeated, and the deeper mount wins whatever the order +expect_url -m $T=site:/uiowa/twin -m $G=site:/uiowa/twin/flex -a '*/*' site:/uiowa/twin/flex 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" +expect_url -m $G=site:/uiowa/twin/flex -m $T=site:/uiowa/twin -a '*/*' site:/uiowa/twin/flex 0 -- \ + "$GB?read.time.min=2026-01-01&read.time.max=2026-01-02&read.time.inter=60" + +# --- error paths, by exit code ---------------------------------------------- + +expect_rc 4 url $G time=2026-04-01,2026-04-01T01:30 # required inter missing +expect_rc 4 url $G 0 foo=1 # no such key +expect_rc 4 url $G 0 fracSecs=4 # ambiguous (das, csv) +expect_rc 4 url $G 0 formats.xml # no such format +expect_rc 4 url $G 0 lshell=maybe # not a boolean +expect_rc 4 url $G 0 delimiter=Colon # not in enum +expect_rc 4 url $G 0 version=das3 # geomag offers only das2 +expect_rc 4 url $G 9 # no such example +expect_rc 4 url $G example=0 # sections are not keys +expect_rc 4 url $G examples.one # example must be a number +expect_rc 4 url $G 0 data.nope # no such data variable +expect_rc 2 node -s bogus $G # bad section list +expect_rc 2 url # missing path +expect_rc 2 bogus $G # unknown command +expect_rc 3 url $FX/does_not_exist.json 0 # unreadable node + +# formats are hidden by default, -s or -v brings them back +if $FETCH node $G 2>/dev/null | grep -q "delimiter"; then + echo " FAIL: das_fetch node $G showed format keys without -s"; nFail=$((nFail + 1)) +else + nPass=$((nPass + 1)) +fi +if $FETCH node -s formats $G 2>/dev/null | grep -q "delimiter"; then + nPass=$((nPass + 1)) +else + echo " FAIL: das_fetch node -s formats $G hid the format keys"; nFail=$((nFail + 1)) +fi + +# node and json commands run on every fixture and print the key table header +for f in $G $H $J; do + if $FETCH node $f 2>/dev/null | grep -q "^ Key "; then + nPass=$((nPass + 1)) + else + echo " FAIL: das_fetch node $f printed no key table" + nFail=$((nFail + 1)) + fi + if $FETCH json "$f#protocol" 2>/dev/null | grep -q '"method"\|"convention"'; then + nPass=$((nPass + 1)) + else + echo " FAIL: das_fetch json $f#protocol printed no protocol block" + nFail=$((nFail + 1)) + fi +done + +if [ $nFail -eq 0 ]; then + echo " Result: PASSED ($nPass checks)" + exit 0 +else + echo " Result: FAILED ($nFail of $((nPass + nFail)) checks)" + exit 1 +fi diff --git a/test/das_walk/a.json b/test/das_walk/a.json new file mode 100644 index 0000000..df77d68 --- /dev/null +++ b/test/das_walk/a.json @@ -0,0 +1,9 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "a", + "separator": "/", + "catalog": { + "b": {"type": "HttpStreamSrc", "urls": ["test/das_walk/src.json"]} + } +} diff --git a/test/das_walk/ab.json b/test/das_walk/ab.json new file mode 100644 index 0000000..a4f9f4a --- /dev/null +++ b/test/das_walk/ab.json @@ -0,0 +1,9 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "ab", + "separator": "/", + "catalog": { + "x": {"type": "HttpStreamSrc", "urls": ["test/das_walk/src.json"]} + } +} diff --git a/test/das_walk/fedroot.json b/test/das_walk/fedroot.json new file mode 100644 index 0000000..4640f14 --- /dev/null +++ b/test/das_walk/fedroot.json @@ -0,0 +1,10 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "", + "title": "Stands in for the federated root: children are named by URI prefix", + "separator": null, + "catalog": { + "tag:das2.org,2012:site:": {"type": "Catalog", "urls": ["test/das_walk/root.json"]} + } +} diff --git a/test/das_walk/junk.json b/test/das_walk/junk.json new file mode 100644 index 0000000..85e4783 --- /dev/null +++ b/test/das_walk/junk.json @@ -0,0 +1 @@ +this is not a node document diff --git a/test/das_walk/root.json b/test/das_walk/root.json new file mode 100644 index 0000000..b8eacbd --- /dev/null +++ b/test/das_walk/root.json @@ -0,0 +1,33 @@ +{ + "type": "Catalog", + "version": "0.5", + "name": "root", + "title": "Walk test root: mirrors, look-alikes, junk and a dead listing", + "separator": "/", + "catalog": { + "a": { + "type": "Catalog", + "title": "First mirror is missing, second works", + "urls": ["test/das_walk/missing.json", "test/das_walk/a.json"] + }, + "ab": { + "type": "Catalog", + "title": "Its name starts with a, so a is tried first for ab/x", + "urls": ["test/das_walk/ab.json"] + }, + "junk": { + "type": "Catalog", + "title": "The file is not JSON", + "urls": ["test/das_walk/junk.json"] + }, + "nourls": { + "type": "Catalog", + "title": "Listed with nowhere to load from", + "urls": [] + }, + "src": { + "type": "HttpStreamSrc", + "urls": ["test/das_walk/src.json"] + } + } +} diff --git a/test/das_walk/src.json b/test/das_walk/src.json new file mode 100644 index 0000000..a47874f --- /dev/null +++ b/test/das_walk/src.json @@ -0,0 +1,8 @@ +{ + "type": "HttpStreamSrc", + "version": "0.7", + "name": "src", + "title": "The leaf every path in this fixture ends at", + "protocol": {"baseUrls": ["http://127.0.0.1:8765/walk"], "httpParams": {}}, + "interface": {} +} diff --git a/utilities/das2_cache_rdr.c b/utilities/das2_cache_rdr.c index 9ca3183..e793d50 100644 --- a/utilities/das2_cache_rdr.c +++ b/utilities/das2_cache_rdr.c @@ -445,8 +445,10 @@ char** timeBinFileList( /* Getting generic binned parameter lists */ /* Get GCC to shut it's cakehole about the unused parameters below */ +#ifdef __GNUC__ /* quiet msvc */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#endif char** generalBinFileList( const char* sRoot, cache_tree_t* pTree, double rBeg, double rEnd, @@ -457,7 +459,9 @@ char** generalBinFileList( return NULL; } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif /* ************************************************************************* */ /* Writing output data */ @@ -560,13 +564,17 @@ DasErrCode onPktData(PktDesc* pPktIn, void* vpHDat){ /* Callback to ignore no data messages, we'll count and make our own */ /* Like I said. GCC, just freaking chill okay */ +#ifdef __GNUC__ /* quiet msvc */ #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunused-parameter" +#endif DasErrCode onIgnoreNoData(OobExcept* pExcept, void* vpHDat) { return DAS_OKAY; } +#ifdef __GNUC__ #pragma GCC diagnostic pop +#endif /* ************************************************************************* */ int readCache( diff --git a/utilities/das2_from_cdf.c b/utilities/das2_from_cdf.c index ed6e5d1..2810543 100644 --- a/utilities/das2_from_cdf.c +++ b/utilities/das2_from_cdf.c @@ -395,7 +395,7 @@ int parseArgs(int argc, char** argv, popts_t* pOpts) strncpy(pOpts->range.sCoord, "time", sizeof(pOpts->range.sCoord) - 1); int nLater = pOpts->nVars; const char* asLater[MAX_DATA_VARS]; - memcpy(asLater, pOpts->asVars, nLater * sizeof(const char*)); + memcpy((void*)asLater, pOpts->asVars, nLater * sizeof(const char*)); /* quiet msvc */ pOpts->nVars = 0; if((sBeg != NULL)&&(_addVars(pOpts, sBeg) != DAS_OKAY)) return PERR; if((sEnd != NULL)&&(_addVars(pOpts, sEnd) != DAS_OKAY)) return PERR; diff --git a/utilities/das3_cdf.c b/utilities/das3_cdf.c index 3e5b551..11244b0 100644 --- a/utilities/das3_cdf.c +++ b/utilities/das3_cdf.c @@ -60,8 +60,8 @@ #define DASCDF_COMP_STR_SZ 32 /* room for one component label or units string */ #define DASCDF_MAX_ATTR_MAP 16 /* --prop-map entries */ -#define DEF_AUTH_FILE ".dasauth" -#define DEF_TEMP_DIR ".dastmp" +#define CRED_FILE "servers" /* under das_userdir() */ +#define DEF_TEMP_DIR "cdftmp" /* under das_userdir(), so one rm -r clears it */ #define LOC_PATH_LEN 256 @@ -69,13 +69,13 @@ #define VAR_MAP_MAX_LINE 256 +/* For help text only, das_userdir() does the lookup */ #ifdef _WIN32 -#define HOME_VAR_STR "USERPROFILE" -#define HELP_TEMP_DIR "%USERPROFILE%\\" DEF_TEMP_DIR +#define HELP_USERDIR "%USERPROFILE%\\.das" #else -#define HOME_VAR_STR "HOME" -#define HELP_TEMP_DIR "$HOME/" DEF_TEMP_DIR +#define HELP_USERDIR "$HOME/.das" #endif +#define HELP_TEMP_DIR HELP_USERDIR DAS_DSEPS DEF_TEMP_DIR /* Add a little user-flag for arrays so we know which ones to clear after a batch write */ @@ -253,11 +253,15 @@ void prnHelp() "\n" " -t DIR,--temp-dir=DIR\n" " Directory for writing temporary files when run as a command\n" -" pipeline filter. Defaults to \"%s\". Ignored if -o is given.\n" +" pipeline filter. Defaults to \"%s\", or the same\n" +" place under DAS_USERDIR if that environment variable is set.\n" +" Use this, or DAS_USERDIR, when running under a web server\n" +" whose account has no writable home. Ignored if -o is given.\n" "\n" " -a FILE,--auth-toks=FILE\n" -" Set the location where server authentication tokens (if any)\n" -" are saved. Defaults to %s%s%s\n" +" Read saved server credentials from FILE. Defaults to\n" +" " HELP_USERDIR DAS_DSEPS CRED_FILE ", or the same file under DAS_USERDIR if that\n" +" environment variable is set.\n" "\n" " -i URL,--input=URL\n" " Instead of reading from standard input, read from a given URL.\n" @@ -338,7 +342,7 @@ void prnHelp() " This is needed for any CDF files submitted to the Planetary Data\n" " system. Per ISTP rules, Epoch variables are not compressed.\n" "\n", -HELP_TEMP_DIR, HOME_VAR_STR, DAS_DSEPS, DEF_AUTH_FILE); +HELP_TEMP_DIR); printf( @@ -424,15 +428,15 @@ int parseArgs(int argc, char** argv, popts_t* pOpts) /* Set a few defaults */ snprintf( - pOpts->aCredFile, DAS_FIELD_SZ(popts_t, aCredFile) - 1, "%s" DAS_DSEPS DEF_AUTH_FILE, - das_userhome() + pOpts->aCredFile, DAS_FIELD_SZ(popts_t, aCredFile) - 1, "%s" DAS_DSEPS CRED_FILE, + das_userdir() ); strcpy(pOpts->aLevel, "info"); snprintf( - pOpts->aTmpDir, DAS_FIELD_SZ(popts_t, aTmpDir) - 1, "%s" DAS_DSEPS ".cdftmp", - das_userhome() + pOpts->aTmpDir, DAS_FIELD_SZ(popts_t, aTmpDir) - 1, "%s" DAS_DSEPS DEF_TEMP_DIR, + das_userdir() ); int i = 0; @@ -3568,10 +3572,11 @@ int main(int argc, char** argv) else{ /* Just writing to a temporary location */ /* Temporary write and stream. Would love to use tmpfile here, but - libcdf doesn't support it */ + libcdf doesn't support it. An explicit -t "" falls back to the + default, parseArgs() fills it otherwise. */ if(opts.aTmpDir[0] == '\0'){ snprintf(ctx.sWriteTo, LOC_PATH_LEN-1, "%s%c%s%cd3cdf-tmp-%d.cdf", - das_userhome(), DAS_DSEPC, DEF_TEMP_DIR, DAS_DSEPC, getpid() + das_userdir(), DAS_DSEPC, DEF_TEMP_DIR, DAS_DSEPC, getpid() ); } else{ @@ -3609,6 +3614,7 @@ int main(int argc, char** argv) ){ pCreds = new_CredMngr(opts.aCredFile); + if(das_isfile(opts.aCredFile)) CredMngr_load(pCreds, NULL, NULL); /* Give it a connection time out of 6 seconds */ if(!das_http_getBody(opts.aSource, "das3_cdf", pCreds, &res, 6.0)){ diff --git a/utilities/das3_from_cdf.c b/utilities/das3_from_cdf.c index b4da0d2..d9ad5fe 100644 --- a/utilities/das3_from_cdf.c +++ b/utilities/das3_from_cdf.c @@ -548,7 +548,7 @@ int parseArgs(int argc, char** argv, popts_t* pOpts) and they came before the ones already added */ int nLater = pOpts->nVars; const char* asLater[MAX_DATA_VARS]; - memcpy(asLater, pOpts->asVars, nLater * sizeof(const char*)); + memcpy((void*)asLater, pOpts->asVars, nLater * sizeof(const char*)); /* quiet msvc */ pOpts->nVars = 0; if((sBeg != NULL)&&(_addVars(pOpts, sBeg) != DAS_OKAY)) return PERR; if((sEnd != NULL)&&(_addVars(pOpts, sEnd) != DAS_OKAY)) return PERR; diff --git a/utilities/das3_info.c b/utilities/das3_info.c index 3df0e9e..05df15e 100644 --- a/utilities/das3_info.c +++ b/utilities/das3_info.c @@ -42,15 +42,13 @@ /* Globals */ #define PERR DASERR_MAX + 1 -#define DEF_AUTH_FILE ".dasauth" +#define CRED_FILE "servers" /* under das_userdir() */ #define PROG "das3_info" #ifndef _WIN32 -#define HOME_VAR "$HOME" -#define HOME_ENV "HOME" +#define HELP_USERDIR "$HOME/.das" /* for help text, das_userdir() does the lookup */ #else -#define HOME_VAR "%%USERPROFILE%%" -#define HOME_ENV "USERPROFILE" +#define HELP_USERDIR "%%USERPROFILE%%\\.das" #endif /* ************************************************************************* */ @@ -92,8 +90,9 @@ void prnHelp() " critical. The default is warning.\n" "\n" "AUTHENTICATION\n" -" URLs that require a login are handled by prompting for credentials,\n" -" which are cached for the session in " HOME_VAR "/" DEF_AUTH_FILE ".\n" +" URLs that require a login prompt for a username and password. Saved\n" +" credentials in " HELP_USERDIR DAS_DSEPS CRED_FILE " are tried first. The environment\n" +" variable DAS_USERDIR replaces " HELP_USERDIR ".\n" "\n" "EXAMPLES\n" " Describe the Cassini/RPWS waveform example provided with the source:\n" @@ -107,7 +106,8 @@ void prnHelp() "SEE ALSO\n" " das3_text, das3_csv\n" "\n" -" The das3 data model is described at https://das2.org/das3\n" +" The das3 data model and this library are documented at\n" +" https://das-developers.github.io/das2C/\n" "\n" ); } @@ -122,15 +122,13 @@ static DasIO* openInput(const char* sInput, float rTimeout) if((strncmp(sInput, "http://", 7) != 0)&&(strncmp(sInput, "https://", 8) != 0)) return new_DasIO_file(PROG, sInput, "r"); - /* A URL: the credential manager only matters if the server asks */ - char* sHome = getenv(HOME_ENV); - if(sHome == NULL){ - das_error(PERR, "Environment variable %s not set", HOME_ENV); - return NULL; - } + /* A URL: the credential manager only matters if the server asks. Saved + credentials are loaded only if the file exists, since CredMngr_load + logs an error for a missing one and most users have none. */ char sCredFile[256] = {'\0'}; - snprintf(sCredFile, 255, "%s%s%s", sHome, DAS_DSEPS, DEF_AUTH_FILE); + snprintf(sCredFile, sizeof(sCredFile), "%s" DAS_DSEPS CRED_FILE, das_userdir()); DasCredMngr* pCreds = new_CredMngr(sCredFile); + if(das_isfile(sCredFile)) CredMngr_load(pCreds, NULL, NULL); DasHttpResp res; memset(&res, 0, sizeof(DasHttpResp)); diff --git a/utilities/das3_node.c b/utilities/das3_node.c deleted file mode 100644 index c644e20..0000000 --- a/utilities/das3_node.c +++ /dev/null @@ -1,212 +0,0 @@ -/* This small utility is mostly meant to demonstrate an interface and thus - * has a liberal open source license... - * - * The MIT License - * - * Copyright 2023 Chris Piker - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#define _POSIX_C_SOURCE 200112L - -#include -#include - -#include -#include -#include - -#define PROG_ERR 64 - -#define AGENT "das2C" - -/* ************************************************************************* */ -void prnHelp() -{ - fprintf(stderr, -"SYNOPSIS\n" -" das_node - Read a node out of the federated catalog\n" -"\n" -"USAGE\n" -" das_node [-h] [-R] [-a ALT_ROOT] [TAG_URI [#FRAGMENT] ]\n" -"\n" -"DESCRIPTION\n" -" das_node is a small utility which resolves a catalog TAG_URI to URL and then\n" -" writes the named catalog node to standard output. By default the\n" -" builtin root nodes are loaded first, then the catalog is walked to find the\n" -" requested sub-node. The walking algorithm automatically backs-up and tries\n" -" alternate branches when URL resolution fails. If there is only one URL for\n" -" a given branch, or if walking all branches still fails to load the node then\n" -" resolution fails.\n" -"\n" -" The TAG_URI may include a document fragment by including the hash symbol and\n" -" then the relative path of the object within the document.\n Paths are clearly\n" -" defined for JSON nodes, but not necessarily for other catalog object formats.\n" -"\n" -" Any node of type Catalog may be a used as the root node. To do so provide\n" -" and absolute URL to the root using '-a' below.\n" -"\n" -"OPTIONS\n" -"\n" -" -h,--help\n" -" Print this help text\n" -"\n" -" -R,--roots\n" -" Print the builtin root URLs and exit\n" -"\n" -" -a URL,--alt-root URL\n" -" Don't use the compiled in root URLs, look for the given object under\n" -" this alternate root catalog object. Useful for testing detached\n" -" catalogs.\n" -"\n" -" -l,--level\n" -" The logging level, one of 'none','crit', 'error', 'warn', 'info', \n" -" 'debug', or 'trace'.\n" -"\n" -"\n" -"EXAMPLES\n" -" Print the compiled in default federated catalog roots:\n" -" das_node -R\n" -"\n" -" Get the U. Iowa Juno site data source catalog:\n" -" das_node tag:das2.org,2012:site:/uiowa/juno\n" -"\n" -" Retrieve a HttpStreamSrc node for Juno Waves Survey data given an\n" -" explicit URL for the root node:\n" -" das_node -a https://das2.org/catalog/das/site/uiowa.json juno/wav/survey/das2\n" -"\n" -" Output only the protocol section of a source catalog object:\n" -" das_node tag:das2.org,2012:site:/uiowa/juno/survey/das2#protocol\n" -"\n" -"AUTHOR\n" -" chris-piker@uiowa.edu\n" -"\n"); -} - -/* ************************************************************************* */ - -int main(int argc, char** argv) -{ - /* Exit on errors, log info messages and above */ - das_init(argv[0], DASERR_DIS_EXIT, 0, DASLOG_INFO, NULL); - - const char* sRootUrl = NULL; - const char* sNodeUri = NULL; - const char* sNodeFrag = NULL; - size_t uRoots = 0; - const char** psRoots; - - for(int i = 1; i < argc; i++){ - if(strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0 ){ - prnHelp(); - return 0; - } - - if(strcmp(argv[i], "-R") == 0 || strcmp(argv[i], "--roots") == 0 ){ - psRoots = das_root_urls(&uRoots); - fprintf(stdout, "Compiled in das federated catalog URLs:\n"); - for(size_t u = 0; u < uRoots; ++u) - fprintf(stdout, " %s\n", psRoots[u]); - return 0; - } - - if(strcmp(argv[i], "-l") == 0 || strcmp(argv[i], "--level") == 0){ - if(argc <= i){ - daslog_error("Logging level argument missing, use -h for help"); - return 4; - } - ++i; - daslog_setlevel(daslog_strlevel(argv[i])); - continue; - } - - if(strcmp(argv[i], "-a") == 0 || strcmp(argv[i], "--alt-root") == 0){ - if(argc <= i){ - daslog_error( - "Alternate root URL missing after -a or --alt-root, use -h for help." - ); - return 5; - } - ++i; - sRootUrl = argv[i]; - continue; - } - - if(sNodeUri == NULL){ - sNodeUri = argv[i]; - char* pHash = strchr(sNodeUri, '#'); - if(pHash != NULL){ - *pHash = '\0'; - sNodeFrag = pHash + 1; - } - continue; - } - - daslog_error_v("Unknown extra parameter: %s\n", argv[i]); - return 3; - } - - DasNode* pRoot; - if(sRootUrl) - pRoot = new_RootNode_url(sRootUrl, NULL, NULL, AGENT); - else - pRoot = new_RootNode(NULL, NULL, AGENT); - - if(pRoot == NULL){ - fprintf(stderr, "ERROR: Couldn't get the root node"); - return 4; - } - - DasNode* pNode; - if(sNodeUri) - pNode = DasNode_subNode(pRoot, sNodeUri, NULL, AGENT); - else - pNode = pRoot; - - if(pNode != NULL){ - fprintf(stderr, - "Loaded node: %s\n" - "From URL: %s\n", - DasNode_name(pNode), DasNode_srcUrl(pNode) - ); - } - else{ - daslog_error_v("Couldn't load %s starting from %s", sNodeUri, DasNode_srcUrl(pRoot)); - return 7; - } - - - if(DasNode_isJson(pNode)){ - if(sNodeFrag != NULL) fprintf(stdout, "Fragment: %s\n", sNodeFrag); - const DasJdo* pJdo = DasNode_getJdo(pNode, sNodeFrag); - const char* pPretty = (char*) DasJdo_writePretty(pJdo, " ", "\n", NULL); - fprintf(stdout, "\n%s\n", pPretty); - } - else{ - fprintf(stdout, - "The object was type %d, there's no printer for it yet.\n", - pRoot->nType - ); - } - - del_RootNode(pRoot); - - return 0; -} diff --git a/utilities/das_fetch.c b/utilities/das_fetch.c new file mode 100644 index 0000000..7720cde --- /dev/null +++ b/utilities/das_fetch.c @@ -0,0 +1,1558 @@ +/* Copyright (C) 2026 Chris Piker + * + * This file is part of das2C, the Core Das C Library. + * + * das2C is free software; you can redistribute it and/or modify it under + * the terms of the GNU Lesser General Public License version 2.1 as published + * by the Free Software Foundation. + * + * das2C is distributed in the hope that it will be useful, but WITHOUT ANY + * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for + * more details. + * + * You should have received a copy of the GNU Lesser General Public License + * version 2.1 along with das2C; if not, see . + */ + +/* das_fetch: walk the das federated catalog, describe data sources, and + * stream their output to stdout, all by catalog path. Nobody types a + * server URL. + * + * Everything that understands a catalog or a data source is the library's + * job: DasCat (das3/catalog.h) resolves paths, DasSrc (das3/source.h) turns + * a source node's "interface" section into keys and a DasReq (das3/req.h) + * of KEY=VALUE settings into the HTTP GET URL the "protocol" section calls + * for. What lives here is the command line spelling of all that: the + * options for where paths start, the example shortcuts, and the listings. + * + * The body of a data response is never inspected. It may be a das2 or das3 + * stream, a CDF file, CSV text, or CCSDS packets. Bytes are bytes. + */ + +#define _POSIX_C_SOURCE 200112L + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#define isatty _isatty +#define fileno _fileno +#define strncasecmp _strnicmp /* MSVC keeps the POSIX one under its own name */ +#else +#include +#include +#include /* strncasecmp */ +#endif + +#define PROG "das_fetch" +#define PERR (DASERR_MAX + 1) + +#define CRED_FILE "servers" /* under das_userdir(); the older ~/.dasauth is not read */ + +#ifndef _WIN32 +#define HOME_VAR "$HOME" /* for help text, das_userdir() does the lookup */ +#else +#define HOME_VAR "%USERPROFILE%" +#endif + +/* Exit codes. Documented in the help text so shell scripts can branch on + * them instead of parsing stderr. */ +#define RET_OK 0 +#define RET_USAGE 2 +#define RET_NODE 3 +#define RET_QUERY 4 +#define RET_SERVER 5 +#define RET_AUTH 6 +#define RET_WRITE 7 + +/* Command line bounds. Identifiers in a node follow the das3 rule enforced + * by das_assert_valid_id(), at most 63 characters, and a full key is + * "section.group.prop", so KEY_SZ comes from that rule, not from a guess. */ +#define MAX_KEYS 512 /* KEY=VALUE arguments on one command line */ +#define MAX_RAW 32 /* -p arguments */ +#define KEY_SZ (8 + 2*DAS_MAX_ID_BUFSZ) +#define NAME_SZ DAS_MAX_ID_BUFSZ +#define URI_SZ 1024 +#define URL_SZ 4096 +#define ERR_SZ 1024 + +#define START_ENV "DASFETCH_START" +#define MOUNT_ENV "DASFETCH_MOUNT" +#define MAX_MOUNTS 16 +#define MAX_KIDS 32 + +/* What to take when a catalog node offers several sources. das3 streams + * are what the rest of the das3 tools read; anything else is still + * better than nothing for a program that only copies bytes to stdout. */ +#define DEF_ACCEPT "application/vnd.das.stream" + +/* ************************************************************************* */ +/* Help */ + +static void prnHelpMain(void) +{ + fputs( +"NAME\n" +" " PROG " - fetch data by das catalog path, no server URLs required\n" +"\n" +"USAGE\n" +" " PROG " [options] seeds\n" +" " PROG " [options] node [PATH]\n" +" " PROG " [options] json PATH[#FRAGMENT]\n" +" " PROG " [options] url PATH [KEY=VALUE ...]\n" +" " PROG " [options] data PATH [KEY=VALUE ...] > out_file\n" +" " PROG " COMMAND -h\n" +"\n" +"DESCRIPTION\n" +" The das federated catalog is a tree of small JSON documents. Catalog\n" +" nodes list other nodes, and data source nodes describe how to query a\n" +" server. " PROG " walks that tree so you never have to know where a\n" +" server lives or what its query string looks like.\n" +"\n" +" Interactive use is 'node' to look around, then 'data' to download. In a\n" +" script, 'data' writes the response body to standard output, which can be\n" +" redirected to a file or piped to das3_csv, das2_psd and friends. The\n" +" body is passed through untouched, so the output format is whatever the\n" +" source produces: das2 or das3 stream, CDF, CSV, or raw packets.\n" +"\n" +"COMMANDS\n" +" seeds Print the compiled in catalog seed URLs.\n" +" node [PATH] List a catalog's children, or a data source's examples\n" +" and query keys. No PATH shows where paths start.\n" +" json PATH Print a node's raw JSON, or a fragment of it.\n" +" url PATH ... Print the HTTP GET URL a query turns into.\n" +" data PATH ... Run the query, write the response body to stdout.\n" +"\n" +" Each command has a help page: '" PROG " data -h' explains query keys,\n" +" '" PROG " node -h' explains the listings.\n" +"\n" +"PATHS\n" +" Paths are catalog paths, not web paths. Any of these forms work:\n" +"\n" +" uiowa/juno/wav/survey relative to the prefix, by\n" +" default '" DASCAT_DEF_PREFIX "'\n" +" site:/uiowa/juno/wav/survey shorthand for the URI form below\n" +" tag:das2.org,2012:site:/uiowa full catalog path URI\n" +" https://host/some/node.json a node by direct URL\n" +" ./local_node.json a node file on disk\n" +"\n" +"OPTIONS\n" +" -h, --help This text, or with a command, that command's help.\n" +" -V, --version Print the program and library version and exit.\n" +"\n" +" -r, --start START Where relative paths start, in one of three forms.\n" +" A catalog URI prefix, default\n" +" '" DASCAT_DEF_PREFIX "':\n" +" -r tag:das2.org,2012:test:/\n" +" A URL or file, the node there becomes the seed, a\n" +" root: paths are relative to it, names below it are\n" +" paths from it, and nothing outside it can be\n" +" reached:\n" +" -r ./my_site_copy.json\n" +" URL=URI, the same with the node declared to be the\n" +" catalog node URI, so that saved requisitions match\n" +" the node the file stands in for. Any -m mounts join\n" +" a seed to make an offline catalog:\n" +" -r ./my_site_copy.json=site:\n" +" -r https://doi.org/10.1037/a0040251/root.json=10.1037/a0040251\n" +" Without -r the environment variable " START_ENV " is\n" +" used the same way, if set.\n" +" -m, --mount URL=NAME\n" +" Mount a node: the node at URL stands in for NAME and\n" +" everything under it, the rest of the catalog is\n" +" unchanged. NAME is a catalog URI, or with a -r URL\n" +" seed, a path from it. A mount does not move where\n" +" relative paths start:\n" +" -m https://tracers-dev.physics.uiowa.edu/stream/source/tracers-1.json=\\\n" +" site:/uiowa/tracers/tracers-1\n" +" May be repeated. The environment variable " MOUNT_ENV "\n" +" stands in for -m, if set.\n" +"\n" +" -s SECTIONS For 'node', which parts of a source to print, a comma\n" +" separated list from: examples, coords, data, formats,\n" +" options, all. The default leaves out formats.\n" +" -p PARAM=VALUE Send a raw HTTP GET parameter, bypassing the interface\n" +" translation. Repeatable.\n" +" -a TYPES Which source to use when a catalog node offers more\n" +" than one: a comma separated list of the media types\n" +" you can read. Order means nothing, any source that\n" +" provides one of them qualifies, and */* is anything.\n" +" Without -a, das3 streams are asked for first and\n" +" then anything, as if by:\n" +" -a '" DEF_ACCEPT "' then -a '*/*'\n" +" If several sources remain, put the name of one on\n" +" the end of the path.\n" +"\n" +" -f Fresh: ignore cached catalog nodes along the path and\n" +" fetch them again. Use after a catalog has changed.\n" +" -C DIR Keep cached catalog nodes in DIR instead of the\n" +" default below, or 'none' for no cache at all.\n" +"\n" +" -t SECONDS Connection timeout for the data request, default 10.\n" +" -q Quiet, only errors on stderr.\n" +" -v Verbose: node and server URLs, all source sections,\n" +" the arguments behind each example, usage hints, and\n" +" a report of each URL contacted.\n" +" -l LEVEL Log level: crit, error, warn, info, debug, trace.\n" +"\n" +"FILES\n" +" " HOME_VAR "/.das/cache/nodes/\n" +" Catalog nodes fetched over HTTP, kept for an hour. Walking a catalog\n" +" costs one request per level, so repeat runs only talk to the data\n" +" server. See -f and -C.\n" +" " HOME_VAR "/.das/" CRED_FILE "\n" +" Saved credentials. When a source needs them these are tried before\n" +" the username and password prompt.\n" +" The environment variable DAS_USERDIR replaces " HOME_VAR "/.das for both.\n" +"\n" +"EXIT CODES\n" +" 0 success 2 usage error 3 path did not resolve 4 bad query\n" +" 5 network or server failure 6 authentication failed 7 write failed\n" +"\n" +"SEE ALSO\n" +" das3_csv, das3_text, das3_info, das2_psd\n" +"\n" +" https://das2.org, and 'json PATH' to see a node as the server sent it.\n" +"\n", stdout); +} + +static void prnHelpNode(void) +{ + fputs( +"USAGE\n" +" " PROG " [options] node [PATH]\n" +"\n" +" Show what is at a catalog path. With no PATH, where paths start, which\n" +" lists the federated sites.\n" +"\n" +"CATALOGS\n" +" A catalog node prints its name and URI, then one line per child:\n" +" name, node type, and title. Append a child's name to the path to step\n" +" down. The URI line ends with the separator this catalog uses, so the\n" +" next path is URI plus child name. A '*' after the type marks node\n" +" types " PROG " can list but not load or fetch from (Das2DSDF, WebSocSrc).\n" +"\n" +" A SourceSet or Collection holds several descriptions of one dataset.\n" +" Only the one 'data' and 'url' would use is shown, chosen by what it\n" +" provides against the -a list. -v lists them all. To use another,\n" +" add its name to the path, as in uiowa/juno/wav/survey/das2.\n" +"\n" +"DATA SOURCES\n" +" An HttpStreamSrc node prints a header, its examples, then one table per\n" +" interface section.\n" +"\n" +" examples Key is the number to give 'data' or 'url' to run that\n" +" example. With -v the KEY=VALUE arguments behind each one\n" +" are shown too.\n" +" coords Coordinate properties, such as the time range.\n" +" data Data variables that can be switched on or off.\n" +" formats Output format groups and their settings. Hidden by\n" +" default, -s formats or -v shows them. The header line\n" +" 'Formats:' always lists the group names.\n" +" options Source options that fit nowhere else.\n" +"\n" +" Table columns: Key is what you type, without the section prefix; Type\n" +" and Units are the value's; Default is what the server uses when the key\n" +" is not given; Req marks keys the server insists on; Values lists an\n" +" enum's choices or a coordinate's range.\n" +"\n" +"OPTIONS\n" +" -s SECTIONS Print only these sections, comma separated: examples,\n" +" coords, data, formats, options, all.\n" +" -v Also print the node and server URLs, all sections, each\n" +" example's arguments, and usage hints.\n" +"\n" +"EXAMPLES\n" +" " PROG " node\n" +" " PROG " node uiowa/juno/wav\n" +" " PROG " node -s examples,coords uiowa/tracers/tracers-2/spice/geomagcoords\n" +" " PROG " -r https://tracers-dev.physics.uiowa.edu/stream/root.json=site:/uiowa/tracers node\n" +"\n", stdout); +} + +static void prnHelpQuery(bool bData) +{ + fprintf(stdout, +"USAGE\n" +" " PROG " [options] %s PATH [KEY=VALUE ...]\n" +"\n", bData ? "data" : "url "); + if(bData) + fputs( +" Resolve the query, request it, and write the response body to stdout\n" +" untouched, so the output is whatever the source produces. Diagnostics\n" +" go to stderr. A password prompt, if the source needs one, is read from\n" +" the terminal, not stdin. Exit code 0 means the whole body arrived.\n" +"\n", stdout); + else + fputs( +" Resolve the query and print the HTTP GET URL that 'data' would request,\n" +" without requesting it. For scripts, and for checking what a KEY=VALUE\n" +" pair turns into.\n" +"\n", stdout); + fputs( +"QUERY KEYS\n" +" Keys come from the source's interface definition, so they differ from\n" +" source to source. Run 'node PATH' to see them. They look like:\n" +"\n" +" coords.time.min=2026-03-01 a coordinate property\n" +" data.lshell.enabled=true a data variable property\n" +" formats.csv.delimiter=Tab an output format property\n" +" options.decompress=true a source option\n" +"\n" +" Shortcuts, as long as the result is unambiguous:\n" +"\n" +" time.min=2026-03-01 drop the section prefix\n" +" time=2026-03-01,2026-03-02 coordinate range MIN,MAX[,STEP]. STEP\n" +" is the resolution if the coordinate has\n" +" one, otherwise the interval. Leave an\n" +" element empty to skip it: time=,2026-03-02\n" +" lshell=true or just lshell toggle a data variable or boolean option\n" +" csv or formats.csv select an output format group\n" +" 1 or examples.1 start from the maintainer's example 1,\n" +" then apply the other keys on top\n" +" min / minimum, max / maximum, res / resolution, inter / interval are\n" +" interchangeable property names\n" +"\n" +" With no keys at all, example 0 runs, so a bare 'data PATH' always\n" +" produces something to look at.\n" +"\n" +" Keys set to their default value are not sent to the server, except for\n" +" parameters the server requires. Required parameters without a default\n" +" must be given or the request is refused locally with a message naming\n" +" the key to set. Nothing in these rules assumes the first coordinate is\n" +" time; the coordinate names are whatever the source declares.\n" +"\n" +" A source that offers both stream versions has a key for it, usually\n" +" version=das2 or version=das3; its default is what the server sends when\n" +" you say nothing.\n" +"\n" +"OPTIONS\n" +" -p PARAM=VALUE Send a raw HTTP GET parameter, bypassing the interface\n" +" translation. Repeatable. Ties your script to one\n" +" server's API, so prefer KEY=VALUE when you can.\n" +" -t SECONDS Connection timeout, default 10.\n" +"\n" +"EXAMPLES\n" +" A day of Juno Waves survey data:\n" +"\n" +" " PROG " data uiowa/juno/wav/survey time=2021-02-21,2021-02-22 \\\n" +" > jno_wav_2021-02-21.d2s\n" +"\n" +" A source's first example, just to see that it works:\n" +"\n" +" " PROG " data uiowa/tracers/tracers-2/spice/geomagcoords | das3_text -3\n" +"\n" +" Predicted TRACERS-2 magnetic coordinates, one point a minute, as CSV:\n" +"\n" +" " PROG " data uiowa/tracers/tracers-2/spice/geomagcoords \\\n" +" time=2026-04-01,2026-04-01T01:30,60 csv\n" +"\n" +" One housekeeping value from a development server, through das3_csv:\n" +"\n" +" " PROG " -r https://tracers-dev.physics.uiowa.edu/stream/root.json=site:/uiowa/tracers \\\n" +" data tracers-2/ql/mag/hsk time=2026-01-01,2026-01-02 sprt | das3_csv\n" +"\n" +" The URL a query turns into, without running it:\n" +"\n" +" " PROG " url uiowa/juno/wav/survey time=2021-02-21,2021-02-22 b\n" +"\n", stdout); +} + +static void prnHelpJson(void) +{ + fputs( +"USAGE\n" +" " PROG " [options] json PATH[#FRAGMENT]\n" +"\n" +" Print the node's JSON document, pretty printed, or just the part at\n" +" FRAGMENT, a slash separated path into it. For looking at what the\n" +" catalog really says when a listing leaves you wondering.\n" +"\n" +"EXAMPLES\n" +" " PROG " json uiowa/juno/wav/survey\n" +" " PROG " json 'uiowa/tracers/tracers-2/spice/geomagcoords#protocol/httpParams'\n" +" " PROG " json 'uiowa/tracers/tracers-2/spice/geomagcoords#interface/examples'\n" +"\n", stdout); +} + +static void prnHelpSeeds(void) +{ + fputs( +"USAGE\n" +" " PROG " seeds\n" +"\n" +" Print the compiled in URLs of the das catalog index, one per line.\n" +" The index is mirrored, the first URL that answers is used. A -r START\n" +" argument or the " START_ENV " variable replaces it, a -m URL=URI argument\n" +" or the " MOUNT_ENV " variable mounts a node on it, see '" PROG " -h'.\n" +"\n", stdout); +} + +/* Help for a command, or the overview */ +static void prnHelp(const char* sCmd) +{ + if(sCmd == NULL) prnHelpMain(); + else if((strcmp(sCmd, "node") == 0)||(strcmp(sCmd, "ls") == 0)) prnHelpNode(); + else if((strcmp(sCmd, "data") == 0)||(strcmp(sCmd, "get") == 0)) prnHelpQuery(true); + else if(strcmp(sCmd, "url") == 0) prnHelpQuery(false); + else if(strcmp(sCmd, "json") == 0) prnHelpJson(); + else if(strcmp(sCmd, "seeds") == 0) prnHelpSeeds(); + else prnHelpMain(); +} + +/* ************************************************************************* */ +/* Small string helpers */ + +static bool strieq(const char* a, const char* b) +{ + if((a == NULL)||(b == NULL)) return false; + while(*a && *b){ + if(tolower((unsigned char)*a) != tolower((unsigned char)*b)) return false; + ++a; ++b; + } + return (*a == '\0') && (*b == '\0'); +} + +static void strcpyz(char* sDest, const char* sSrc, size_t uDest) +{ + if(uDest == 0) return; + if(sSrc == NULL){ sDest[0] = '\0'; return; } + strncpy(sDest, sSrc, uDest - 1); + sDest[uDest - 1] = '\0'; +} + +static bool isDigits(const char* s) +{ + if((s == NULL)||(*s == '\0')) return false; + for(; *s; ++s) if(!isdigit((unsigned char)*s)) return false; + return true; +} + +/* A string element of a node document, or NULL. Only the listings read + * documents directly, everything about a source comes through DasSrc. */ +static const char* jstr(const DasJdo* p, const char* sPath) +{ + return (p == NULL) ? NULL : DasJdo_string(DasJdo_get(p, sPath)); +} + +/* Section names as typed in -s lists, singular or plural */ +static bool secAlias(const char* s, das_src_sec_e* pSec) +{ + if(strieq(s,"coord")||strieq(s,"coords")||strieq(s,"coordinate")||strieq(s,"coordinates")){ + *pSec = das_src_sec_coord; return true; + } + if(strieq(s,"data")){ *pSec = das_src_sec_data; return true; } + if(strieq(s,"format")||strieq(s,"formats")||strieq(s,"fmt")){ *pSec = das_src_sec_format; return true; } + if(strieq(s,"option")||strieq(s,"options")||strieq(s,"opt")||strieq(s,"opts")){ + *pSec = das_src_sec_option; return true; + } + return false; +} + +/* ************************************************************************* */ +/* Display settings and terminal wrapping */ + +/* Which parts of a source 'node' prints: one bit per section plus examples */ +#define SHOW_EXAMPLES (1u << das_src_sec_size) +#define SHOW_ALL ((1u << (das_src_sec_size + 1)) - 1) +static unsigned g_uShow = SHOW_ALL & ~(1u << das_src_sec_format); +static bool g_bVerbose = false; +static int g_nCols = 0; /* terminal width, 0 when stdout is not a terminal */ + +/* Display width of UTF-8 text, code points not bytes. Catalog text has no + * double width characters in practice, so this is as far as it goes. */ +static size_t u8width(const char* s, size_t uBytes) +{ + size_t w = 0; + for(size_t u = 0; u < uBytes; ++u) if((((unsigned char)s[u]) & 0xC0) != 0x80) ++w; + return w; +} + +/* Print text that starts at column nCol, breaking on spaces at the terminal + * width with continuation lines indented back to nCol. Without a terminal, + * or with too little room to be worth it, the text goes out as is. */ +static void fputsWrappedAt(const char* sText, int nCol, int nWidth, FILE* pOut) +{ + int nAvail = nWidth - nCol; + if((nWidth <= 0)||(nAvail < 20)||(u8width(sText, strlen(sText)) <= (size_t)nAvail)){ + fputs(sText, pOut); + return; + } + const char* p = sText; + bool bFirst = true; + while(*p != '\0'){ + while(*p == ' ') ++p; + if(*p == '\0') break; + const char* pEnd = p; + const char* pBreak = NULL; + size_t w = 0; + while((*pEnd != '\0') && (w < (size_t)nAvail)){ + if(*pEnd == ' ') pBreak = pEnd; + if((((unsigned char)*pEnd) & 0xC0) != 0x80) ++w; + ++pEnd; + } + if((*pEnd == '\0')||(*pEnd == ' ')) pBreak = pEnd; /* fits, or breaks right here */ + else if(pBreak == NULL) pBreak = pEnd; /* one long word, cut it */ + if(!bFirst){ + fputc('\n', pOut); + for(int i = 0; i < nCol; ++i) fputc(' ', pOut); + } + fwrite(p, 1, (size_t)(pBreak - p), pOut); + bFirst = false; + p = pBreak; + } +} + +static void fputsWrapped(const char* sText, int nCol, FILE* pOut) +{ + fputsWrappedAt(sText, nCol, g_nCols, pOut); +} + +/* A paragraph of explanation, wrapped to the terminal, or to 80 columns + * when there is none so piped output does not get one endless line. An + * optional indent applies to every line. */ +static void prnPara(FILE* pOut, int nIndent, const char* sText) +{ + for(int i = 0; i < nIndent; ++i) fputc(' ', pOut); + fputsWrappedAt(sText, nIndent, (g_nCols > 0) ? g_nCols : 80, pOut); + fputc('\n', pOut); +} + +/* A labeled header line, wrapped to the terminal */ +static void prnField(FILE* pOut, const char* sLabel, const char* sText) +{ + if((sText == NULL)||(sText[0] == '\0')) return; + fprintf(pOut, "%-10s", sLabel); + fputsWrapped(sText, 10, pOut); + fputc('\n', pOut); +} + +/* Fixed properties are shown dimmed under -v so the eye goes to what can be + * set. Only on a terminal; a pipe gets plain text and the Req column + * says "fixed" either way. */ +static const char* dimOn(void){ return (g_nCols > 0) ? "\033[2m" : ""; } +static const char* dimOff(void){ return (g_nCols > 0) ? "\033[0m" : ""; } + +/* ************************************************************************* */ +/* Describing a source. The library has the facts, these functions only + * decide what goes in which column. */ + +/* The node's spelling of a type when it gave one, else a name for what + * the library inferred */ +static const char* propTypeName(const DasSrcProp* p) +{ + const char* s = DasSrcProp_typeName(p); + if((s != NULL)&&(s[0] != '\0')) return s; + switch(DasSrcProp_type(p)){ + case das_srcprop_bool: return "boolean"; + case das_srcprop_int: return "integer"; + case das_srcprop_real: return "real"; + case das_srcprop_time: return "isotime"; + case das_srcprop_enum: return "enum"; + default: return "string"; + } +} + +/* Allowed values for display: enum entries, a range, or true/false */ +static void propValues(const DasSrcProp* p, char* sBuf, size_t uLen) +{ + sBuf[0] = '\0'; + const DasSrcCtrl* pC = DasSrcProp_ctrl(p); + if(pC == NULL) return; + + size_t uEnum = DasSrcCtrl_enumCount(pC); + if(uEnum > 0){ + size_t u = 0; + for(size_t i = 0; i < uEnum; ++i){ + const char* sVal = DasSrcCtrl_enum(pC, i); + if(u + strlen(sVal) + 3 >= uLen){ strcpyz(sBuf + u, "...", uLen - u); return; } + u += snprintf(sBuf + u, uLen - u, "%s%s", (u > 0) ? ", " : "", sVal); + } + return; + } + const char* sLo = ""; + const char* sHi = ""; + if(DasSrcCtrl_range(pC, &sLo, &sHi)){ + snprintf(sBuf, uLen, "%s to %s", sLo, sHi); + return; + } + if(DasSrcProp_type(p) == das_srcprop_bool) strcpyz(sBuf, "true, false", uLen); +} + +/* Print the arguments that reproduce an example, indented under its row. + * The library already has them in interface terms, whatever the node + * layout, so this is a requisition read back as KEY=VALUE pairs. */ +static void printExampleArgs(const DasSrc* pSrc, size_t uIdx, int nCol, FILE* pOut) +{ + DasReq* pReq = DasSrc_newReq(pSrc, (int)uIdx); + if(pReq == NULL) return; + char sArgs[4096] = {'\0'}; + size_t u = 0; + for(size_t i = 0; (i < DasReq_count(pReq)) && (u < sizeof(sArgs)); ++i) + u += snprintf(sArgs + u, sizeof(sArgs) - u, "%s=%s ", DasReq_key(pReq, i), DasReq_val(pReq, i)); + for(size_t i = 0; (i < DasReq_rawCount(pReq)) && (u < sizeof(sArgs)); ++i){ + const char* sVal = NULL; + const char* sName = DasReq_raw(pReq, i, &sVal); + u += snprintf(sArgs + u, sizeof(sArgs) - u, "-p %s=%s ", sName, sVal); + } + del_DasReq(pReq); + + for(int i = 0; i < nCol; ++i) fputc(' ', pOut); + fputsWrapped(sArgs, nCol, pOut); + fputs("\n", pOut); +} + +/* The examples table: number and what it provides, arguments only with -v */ +static void printExamples(const DasSrc* pSrc, FILE* pOut) +{ + size_t uEx = DasSrc_exampleCount(pSrc); + if(uEx == 0) return; + size_t wKey = 3; + char sNum[16]; + for(size_t u = 0; u < uEx; ++u){ + snprintf(sNum, sizeof(sNum), "%zu", u); + if(strlen(sNum) > wKey) wKey = strlen(sNum); + } + fprintf(pOut, "\nexamples:\n %-*s Provides\n ", (int)wKey, "Key"); + for(size_t i = 0; i < wKey; ++i) fputc('-', pOut); + fputs(" --------\n", pOut); + int nCol = 2 + (int)wKey + 2; + for(size_t u = 0; u < uEx; ++u){ + fprintf(pOut, " %-*zu ", (int)wKey, u); + fputsWrapped(DasSrc_exampleLabel(pSrc, u), nCol, pOut); + fputc('\n', pOut); + if(g_bVerbose) printExampleArgs(pSrc, u, nCol, pOut); + } +} + +/* The realm and methods are not modeled by DasSrc, they come from the + * protocol section as the node wrote it. Old nodes say "authentication". */ +static void prnAuth(const DasSrc* pSrc, FILE* pOut) +{ + const DasJdo* pAuth = DasSrc_jdo(pSrc, "protocol/authorization"); + if(pAuth == NULL) pAuth = DasSrc_jdo(pSrc, "protocol/authentication"); + if(pAuth == NULL){ fputs("Auth: unknown\n", pOut); return; } + if(!DasSrc_needsAuth(pSrc)){ fputs("Auth: not required\n", pOut); return; } + fputs("Auth: REQUIRED", pOut); + if(jstr(pAuth, "realm")) fprintf(pOut, ", realm \"%s\"", jstr(pAuth, "realm")); + const DasJdo* pMeth = DasJdo_get(pAuth, "methods"); + if((pMeth != NULL)&&(pMeth->type == das_json_type_ary)){ + fputs(" via", pOut); + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pMeth); pEl; pEl = pEl->next) + fprintf(pOut, " %s", DasJdo_string(pEl->value) ? DasJdo_string(pEl->value) : "?"); + } + if(jstr(pAuth, "description")) fprintf(pOut, "\n %s", jstr(pAuth, "description")); + fputs("\n", pOut); +} + +/* The format groups by name, or for a source without any the one MIME + * type it produces */ +static void prnFormats(const DasSrc* pSrc, FILE* pOut) +{ + size_t uGroups = DasSrc_groupCount(pSrc, das_src_sec_format); + if(uGroups > 0){ + fputs("Formats: ", pOut); + for(size_t g = 0; g < uGroups; ++g){ + const DasSrcGroup* pG = DasSrc_group(pSrc, das_src_sec_format, g); + fprintf(pOut, " %s", DasSrcGroup_name(pG)); + const DasSrcProp* pOn = DasSrcGroup_enable(pG); + if((pOn != NULL) && strieq(DasSrcProp_value(pOn), "true")) fputs(" (default)", pOut); + } + fputs("\n", pOut); + return; + } + const char* sMime = DasSrc_mime(pSrc, NULL); + if(sMime[0] != '\0') fprintf(pOut, "Formats: %s\n", sMime); +} + +/* One table per section. Keys are shown without their section prefix + * since the heading carries it. Column widths are per section so the long + * enum lists of the formats section do not push everything else over. + * Fixed properties, the ones with no control, only appear under -v. */ +static void printKeyTables(const DasSrc* pSrc, FILE* pOut) +{ + enum { C_KEY, C_TYPE, C_UNITS, C_DEF, C_REQ, C_VALS, C_DESC, C_COUNT }; + const char* aHdr[C_COUNT] = {"Key", "Type", "Units", "Default", "Req", "Values", "Description"}; + char sVals[80]; + char sTitle[64]; + + for(int iSec = 0; iSec < das_src_sec_size; ++iSec){ + das_src_sec_e sec = (das_src_sec_e)iSec; + if(!(g_uShow & (1u << sec))) continue; + + /* First pass: is there anything to show, and how wide is it */ + size_t aW[C_COUNT]; + for(int c = 0; c < C_COUNT; ++c) aW[c] = strlen(aHdr[c]); + size_t uRows = 0; + for(size_t g = 0; g < DasSrc_groupCount(pSrc, sec); ++g){ + const DasSrcGroup* pG = DasSrc_group(pSrc, sec, g); + for(size_t i = 0; i < DasSrcGroup_propCount(pG); ++i){ + const DasSrcProp* p = DasSrcGroup_prop(pG, i); + if((DasSrcProp_ctrl(p) == NULL) && !g_bVerbose) continue; + ++uRows; + const char* sKey = DasSrcProp_key(p); + const char* sShort = strchr(sKey, '.') ? strchr(sKey, '.') + 1 : sKey; + const char* sDef = DasSrcProp_value(p); + propValues(p, sVals, sizeof(sVals)); + size_t a[C_COUNT] = { + strlen(sShort), strlen(propTypeName(p)), strlen(DasSrcProp_units(p)), + sDef ? strlen(sDef) : 0, DasSrcProp_ctrl(p) ? 3 : 5, strlen(sVals), 0 + }; + for(int c = 0; c < C_DESC; ++c) if(a[c] > aW[c]) aW[c] = a[c]; + } + } + if(uRows == 0) continue; + int nDescCol = 2; + for(int c = 0; c < C_DESC; ++c) nDescCol += (int)aW[c] + 2; + + fprintf(pOut, "\n%s:\n", das_src_sec_name(sec)); + fprintf(pOut, " %-*s %-*s %-*s %-*s %-*s %-*s %s\n", + (int)aW[C_KEY], aHdr[C_KEY], (int)aW[C_TYPE], aHdr[C_TYPE], + (int)aW[C_UNITS], aHdr[C_UNITS], (int)aW[C_DEF], aHdr[C_DEF], + (int)aW[C_REQ], aHdr[C_REQ], (int)aW[C_VALS], aHdr[C_VALS], aHdr[C_DESC]); + fputs(" ", pOut); + for(int c = 0; c < C_COUNT; ++c){ + size_t w = (c == C_DESC) ? 24 : aW[c]; + for(size_t i = 0; i < w; ++i) fputc('-', pOut); + if(c < C_DESC) fputs(" ", pOut); + } + fputs("\n", pOut); + + for(size_t g = 0; g < DasSrc_groupCount(pSrc, sec); ++g){ + const DasSrcGroup* pG = DasSrc_group(pSrc, sec, g); + for(size_t i = 0; i < DasSrcGroup_propCount(pG); ++i){ + const DasSrcProp* p = DasSrcGroup_prop(pG, i); + const DasSrcCtrl* pC = DasSrcProp_ctrl(p); + if((pC == NULL) && !g_bVerbose) continue; + const char* sKey = DasSrcProp_key(p); + const char* sShort = strchr(sKey, '.') ? strchr(sKey, '.') + 1 : sKey; + const char* sDef = DasSrcProp_value(p); + const char* sReq = (pC == NULL) ? "fixed" : DasSrcCtrl_required(pC) ? "yes" : ""; + propValues(p, sVals, sizeof(sVals)); + if(pC == NULL) fputs(dimOn(), pOut); + fprintf(pOut, " %-*s %-*s %-*s %-*s %-*s %-*s ", + (int)aW[C_KEY], sShort, (int)aW[C_TYPE], propTypeName(p), + (int)aW[C_UNITS], DasSrcProp_units(p), (int)aW[C_DEF], sDef ? sDef : "", + (int)aW[C_REQ], sReq, (int)aW[C_VALS], sVals); + fputsWrapped(DasSrcProp_title(p, sTitle, sizeof(sTitle)), nDescCol, pOut); + if(pC == NULL) fputs(dimOff(), pOut); + fputc('\n', pOut); + } + } + } +} + +/* Shortcut hints specific to this source: the coordinates that take a + * MIN,MAX[,STEP] range */ +static void printShortcuts(const DasSrc* pSrc, FILE* pOut) +{ + fputs("\nShortcuts:\n", pOut); + for(size_t g = 0; g < DasSrc_groupCount(pSrc, das_src_sec_coord); ++g){ + const DasSrcGroup* pG = DasSrc_group(pSrc, das_src_sec_coord, g); + const DasSrcProp* pMin = DasSrcGroup_findProp(pG, "min"); + const DasSrcProp* pMax = DasSrcGroup_findProp(pG, "max"); + if((pMin == NULL)||(pMax == NULL)||(DasSrcProp_ctrl(pMin) == NULL)||(DasSrcProp_ctrl(pMax) == NULL)) + continue; + const DasSrcProp* pRes = DasSrcGroup_findProp(pG, "res"); + const DasSrcProp* pInt = DasSrcGroup_findProp(pG, "inter"); + const char* sStep = (pRes && DasSrcProp_ctrl(pRes)) ? "RES" : + (pInt && DasSrcProp_ctrl(pInt)) ? "INTER" : NULL; + fprintf(pOut, " %s=MIN,MAX%s%s\n", DasSrcGroup_name(pG), sStep ? "," : "", sStep ? sStep : ""); + } + prnPara(pOut, 2, "KEY alone sets a boolean key true, N alone starts from example N, and the " + "section prefix (coords., data., ...) may be dropped when unambiguous. " + "See '" PROG " data -h'."); +} + +static void describeSrc(const DasSrc* pSrc, FILE* pOut, const char* sUri) +{ + const char* sVer = DasSrc_version(pSrc); + fprintf(pOut, "HttpStreamSrc%s%s: %s\n", sVer[0] ? " " : "", sVer, DasSrc_name(pSrc)); + prnField(pOut, "Title:", DasSrc_title(pSrc)); + prnField(pOut, "Summary:", DasJdo_string(DasSrc_jdo(pSrc, D2FRAG_DESC))); + prnField(pOut, "URI:", sUri); + if(g_bVerbose){ + prnField(pOut, "From:", DasSrc_url(pSrc)); + size_t uBases = DasStreamSrc_urlCount(pSrc); + if(uBases > 0){ + fprintf(pOut, "Server: %s", DasStreamSrc_baseUrl(pSrc, 0)); + if(uBases > 1) fprintf(pOut, " (+%zu alternate)", uBases - 1); + fputs("\n", pOut); + } + } + prnAuth(pSrc, pOut); + prnFormats(pSrc, pOut); + + if(g_uShow & SHOW_EXAMPLES) printExamples(pSrc, pOut); + + size_t uCtrls = 0; + for(size_t u = 0; u < DasSrc_propCount(pSrc); ++u) + if(DasSrcProp_ctrl(DasSrc_prop(pSrc, u)) != NULL) ++uCtrls; + if(uCtrls == 0){ + fputs("\nThis source takes no query parameters.\n", pOut); + if(!g_bVerbose) return; + } + else if(g_bVerbose){ + fputs("\nQuery keys, use as KEY=VALUE with 'data' or 'url':\n", pOut); + } + + printKeyTables(pSrc, pOut); + + /* Usage hints are for the first dozen runs, not the hundredth. They + live in 'data -h' and come back with -v. */ + if(g_bVerbose && (uCtrls > 0)) printShortcuts(pSrc, pOut); +} + +/* ************************************************************************* */ +/* Listing a catalog */ + +static void listCatalog(const DasNode* pNode, FILE* pOut, const char* sUri) +{ + const char* sType = jstr(DasNode_getJdo(pNode, NULL), D2FRAG_TYPE); + const char* sVer = DasNode_version(pNode); + fprintf(pOut, "%s%s%s: %s\n", sType ? sType : "Node", sVer ? " " : "", sVer ? sVer : "", + DasNode_label(pNode) ? DasNode_label(pNode) : ""); + prnField(pOut, "Title:", DasNode_title(pNode)); + prnField(pOut, "URI:", sUri); + if(g_bVerbose) prnField(pOut, "From:", DasNode_srcUrl(pNode)); + + size_t uKids = DasNode_subCount(pNode); + if(uKids == 0){ fputs("\nNo children listed.\n", pOut); return; } + + size_t wName = 4, wType = 4; + for(size_t u = 0; u < uKids; ++u){ + const char* sName = DasNode_subName(pNode, u); + const char* sT = jstr(DasNode_subStub(pNode, u), D2FRAG_TYPE); + if(sName && (strlen(sName) > wName)) wName = strlen(sName); + if(sT && (strlen(sT) > wType)) wType = strlen(sT); + } + int nDescCol = 2 + (int)wName + 2 + (int)wType + 3; + + fprintf(pOut, "\nChildren (%zu):\n\n", uKids); + bool bUnsupported = false; + for(size_t u = 0; u < uKids; ++u){ + const char* sName = DasNode_subName(pNode, u); + const DasJdo* pStub = DasNode_subStub(pNode, u); + const char* sT = jstr(pStub, D2FRAG_TYPE); + const char* sDesc = jstr(pStub, D2FRAG_TITLE); + if(sDesc == NULL) sDesc = jstr(pStub, D2FRAG_LABEL); + if(sDesc == NULL) sDesc = jstr(pStub, D2FRAG_DESC); + bool bKnown = (DasNode_subType(pNode, u) != d2node_inv); + if(!bKnown) bUnsupported = true; + fprintf(pOut, " %-*s %-*s%s ", (int)wName, sName ? sName : "?", + (int)wType, sT ? sT : "?", bKnown ? " " : "*"); + fputsWrapped(sDesc ? sDesc : "", nDescCol, pOut); + fputc('\n', pOut); + } + if(bUnsupported) + { + fputc('\n', pOut); + prnPara(pOut, 2, "* this node type can be listed but not loaded or fetched by " PROG); + } +} + +/* What to show as a node's name. A root has no URI: the caller's + * fallback is shown when it has one, the location the node was loaded + * from, else the node is called what it is. Catalogs with a URI get + * their child separator appended so the next step is obvious. */ +static void nodeUri( + const DasNode* pNode, const char* sFallback, char* sBuf, size_t uLen +){ + const char* sUri = DasNode_pathUri(pNode); + if((sUri == NULL)||(sUri[0] == '\0')){ + if((sFallback != NULL) && (sFallback[0] != '\0')) snprintf(sBuf, uLen, "(the root, %s)", sFallback); + else snprintf(sBuf, uLen, "(the root)"); + return; + } + const char* sSep = DasNode_isCatalog(pNode) ? DasNode_pathSep(pNode) : NULL; + snprintf(sBuf, uLen, "%s%s", sUri, sSep ? sSep : ""); +} + +/* ************************************************************************* */ +/* Catalog navigation + * + * DasCat (das3/catalog.h) owns the seed, the mounts, the credentials, the + * agent and the memory of every node loaded, and resolves paths under one + * policy. What stays here is the command line spelling of that policy + * and the wording of a failed lookup. */ + + +/* A -r START or -m URL=URI value, or their environment stand-ins, turned + * into catalog policy. URL=URI names a node: the seed for -r, a mount for + * -m. For -r a URL alone is a seed with no name, a root, and anything + * else is a URI prefix. A mount always needs a name. + * + * The split is at the last '='. What follows it is a name when it has a + * scheme, "site:/x", or, since a mount under a root seed is named by a + * path, when it is not a location and the URL before it carries no query + * string. A URL with a query string can end in "=x" itself, so a mount + * on one needs a name with a scheme. Command line mistakes are reported + * here, the catalog logs its refusals. False on either. */ +static bool applyStartArg(DasCat* pCat, const char* sVal, bool bMount) +{ + const char* sOpt = bMount ? "-m" : "-r"; + const char* pEq = strrchr(sVal, '='); + const char* sName = (pEq != NULL) ? pEq + 1 : NULL; + bool bNamed = false; + if((sName != NULL) && (sName[0] != '\0') && !das_isUrlOrFile(sName)){ + if(das_isUri(sName)) bNamed = true; + else bNamed = (memchr(sVal, '?', (size_t)(pEq - sVal)) == NULL); + } + + if(bNamed){ + char sUrl[URL_SZ]; + size_t uLen = (size_t)(pEq - sVal); + if(uLen >= sizeof(sUrl)) uLen = sizeof(sUrl) - 1; + memcpy(sUrl, sVal, uLen); + sUrl[uLen] = '\0'; + if(!das_isUrlOrFile(sUrl)){ + fprintf(stderr, PROG ": %s takes URL=URI, '%s' before the '=' is not a URL or a file\n", sOpt, sUrl); + return false; + } + if(bMount) return DasCat_mount(pCat, sUrl, sName) == DAS_OKAY; + return DasCat_setSeed(pCat, sUrl, sName) == DAS_OKAY; + } + + /* A location after the '=' is the old order, URI=URL, or "=URL" */ + if((sName != NULL) && das_isUrlOrFile(sName)){ + fprintf(stderr, PROG ": %s takes URL=URI, the location first and the catalog name " + "it stands for after the '='\n", sOpt); + return false; + } + if(bMount){ + fprintf(stderr, PROG ": -m needs URL=URI, see -h\n"); + return false; + } + if(das_isUrlOrFile(sVal)) return DasCat_setSeed(pCat, sVal, NULL) == DAS_OKAY; /* a root */ + if(pEq != NULL){ + fprintf(stderr, PROG ": -r takes URL=URI, and '%s' after the '=' is not a catalog name\n", sName); + return false; + } + return DasCat_setPrefix(pCat, sVal) == DAS_OKAY; +} + +/* Does a child's catalog entry provide any media type in the list? The + list is a set of exact types compared ignoring case, and the bare + wildcard, star slash star, takes anything, including an entry with no + provides list at all. Parameters on the catalog side are cut. */ +static bool stubProvides(const DasJdo* pStub, const char* sAccept) +{ + const DasJdo* pProv = (pStub == NULL) ? NULL : DasJdo_get(pStub, "provides"); + bool bList = (pProv != NULL) && (pProv->type == das_json_type_ary); + + const char* sWant = sAccept; + while(true){ + sWant += strspn(sWant, ", \t"); + size_t uWant = strcspn(sWant, ", \t"); + if(uWant == 0) return false; + if((uWant == 3) && (strncmp(sWant, "*/*", 3) == 0)) return true; + if(bList){ + for(const das_json_ary_el* pEl = DasJdo_aryFirst(pProv); pEl; pEl = pEl->next){ + const char* sHas = DasJdo_string(pEl->value); + if(sHas == NULL) continue; + size_t uHas = strcspn(sHas, "; \t"); + if((uHas == uWant) && (strncasecmp(sWant, sHas, uWant) == 0)) return true; + } + } + sWant += uWant; + } +} + +static const DasJdo* childStub(const DasNode* pNode, const char* sName) +{ + size_t uKids = DasNode_subCount(pNode); + for(size_t u = 0; u < uKids; ++u) + if(strcmp(DasNode_subName(pNode, u), sName) == 0) return DasNode_subStub(pNode, u); + return NULL; +} + +/* The library lists the children it can load; which one to fetch from + * is this program's call, made on the media types each child provides. + * Without -a it wants das3 streams and settles for anything, two asks. + * An -a list is one ask and a set, so a user who lists two formats gets + * every source that emits either and names one on the path if that is + * several. The chosen child is loaded by the session so credentials and + * the cache follow. */ +static DasSrc* pickSrc(DasCat* pCat, DasNode* pNode, const char* sAccept) +{ + if(DasNode_isStreamSrc(pNode)) return new_DasSrc_node(pNode); + + const char* aNames[MAX_KIDS]; + size_t uKids = dassrc_loadable(pNode, aNames, MAX_KIDS); + if(uKids == 0){ + fprintf(stderr, PROG ": nothing under %s is a source this program can fetch from\n", + DasNode_pathUri(pNode)); + return NULL; + } + if(uKids > MAX_KIDS) uKids = MAX_KIDS; + + const char* aAsk[2] = { DEF_ACCEPT, "*/*" }; + size_t uAsks = 2; + if(sAccept != NULL){ aAsk[0] = sAccept; uAsks = 1; } + + const char* aPick[MAX_KIDS]; + size_t uPick = 0; + const char* sAsk = NULL; + for(size_t a = 0; (a < uAsks) && (uPick == 0); ++a){ + sAsk = aAsk[a]; + for(size_t k = 0; k < uKids; ++k) + if(stubProvides(childStub(pNode, aNames[k]), sAsk)) aPick[uPick++] = aNames[k]; + } + + if(uPick != 1){ + if(uPick == 0) + fprintf(stderr, PROG ": nothing under %s provides %s, the sources it has are:", DasNode_pathUri(pNode), sAsk); + else + fprintf(stderr, PROG ": %s offers %zu sources for %s, put one on the end of the path:", DasNode_pathUri(pNode), uPick, sAsk); + const char** aShow = (uPick == 0) ? aNames : aPick; + size_t uShow = (uPick == 0) ? uKids : uPick; + for(size_t u = 0; u < uShow; ++u) fprintf(stderr, " %s", aShow[u]); + fputc('\n', stderr); + return NULL; + } + + DasNode* pSub = DasCat_subNode(pCat, pNode, aPick[0]); + if(pSub == NULL){ + fprintf(stderr, PROG ": couldn't load source '%s' under %s\n", aPick[0], DasNode_pathUri(pNode)); + return NULL; + } + return new_DasSrc_node(pSub); +} + +/* A path did not resolve. Say whether the policy or the catalog is to + * blame, and for the catalog, which child was the problem and what its + * parent offers. Call right after the failed DasCat_resolve(). */ +static void explainUnresolved(DasCat* pCat, const char* sPath) +{ + char sFullUri[URI_SZ]; + strcpyz(sFullUri, DasCat_lastUri(pCat), sizeof(sFullUri)); + + /* The common mistake with a seed is asking for something it does not + cover, say so instead of hunting for a parent */ + if(!DasCat_covers(pCat, sFullUri)){ + daslog_error_v("'%s' is outside the -r seed %s, use -m to mount it on the " + "federated catalog instead", sFullUri, DasCat_prefix(pCat)); + return; + } + + const char* sRest = NULL; + DasNode* pNode = DasCat_nearest(pCat, &sRest); + if((pNode == NULL)||(sRest == NULL)||(sRest[0] == '\0')){ + /* A root seed has no URI, so the path to it is empty; name the file */ + if((sFullUri[0] == '\0') && DasCat_hasSeed(pCat)) + daslog_error_v("Couldn't load the seed %s", DasCat_seedUrl(pCat)); + else + daslog_error_v("Couldn't resolve catalog path '%s'", sFullUri); + return; + } + const char* sParentUri = DasCat_lastUri(pCat); + + /* Only the first component below the deepest node that loaded can be + the culprit */ + char sChild[NAME_SZ]; + strcpyz(sChild, sRest, sizeof(sChild)); + char* pSlash = strchr(sChild, '/'); + if(pSlash != NULL) *pSlash = '\0'; + + char sMsg[ERR_SZ]; + size_t uKids = DasNode_subCount(pNode); + size_t n = 0; + bool bFound = false; + for(size_t u = 0; u < uKids; ++u){ + const char* sName = DasNode_subName(pNode, u); + if(strieq(sName, sChild)){ + const char* sType = jstr(DasNode_subStub(pNode, u), D2FRAG_TYPE); + if(DasNode_subType(pNode, u) == d2node_inv) + n = snprintf(sMsg, sizeof(sMsg), "'%s' under %s is a %s node, which " PROG + " can list but not load or fetch from", sChild, sParentUri, sType ? sType : "?"); + else + n = snprintf(sMsg, sizeof(sMsg), "'%s' under %s is listed but could not be loaded", + sChild, sParentUri); + bFound = true; + break; + } + } + if(!bFound) n = snprintf(sMsg, sizeof(sMsg), "No child named '%s' under %s", sChild, sParentUri); + if(uKids > 0){ + n += snprintf(sMsg + n, sizeof(sMsg) - n, ". Children:"); + for(size_t u = 0; (u < uKids) && (n < sizeof(sMsg)); ++u) + n += snprintf(sMsg + n, sizeof(sMsg) - n, " %s", DasNode_subName(pNode, u)); + } + daslog_error_v("%s", sMsg); +} + +/* Everything main() owns. The catalog frees its nodes, then the credential + * manager they borrow. Returns its argument so exits read as one line. */ +static int finish(DasCat* pCat, int nRet) +{ + del_DasCat(pCat); + return nRet; +} + +/* ************************************************************************* */ +/* Fetching */ + +static int streamUrl(DasCat* pCat, const char* sUrl, float rTimeout) +{ + DasHttpResp res; + memset(&res, 0, sizeof(DasHttpResp)); + + daslog_info_v("Requesting %s", sUrl); + + if(!das_http_getBody(sUrl, DasCat_agent(pCat), DasCat_credMngr(pCat), &res, rTimeout)){ + int nRet = RET_SERVER; + if((res.nCode == 401)||(res.nCode == 403)) nRet = RET_AUTH; + else if(res.nCode == 400) nRet = RET_QUERY; + daslog_error_v("%s", res.sError ? res.sError : "Request failed"); + if(res.nCode == 400) + daslog_error("The server rejected the query. Check the keys with '" + PROG " node' or try the URL from '" PROG " url' in a browser"); + DasHttpResp_freeFields(&res); + return nRet; + } + + char sFinal[URL_SZ]; + das_url_toStr(&(res.url), sFinal, sizeof(sFinal)); + if(strcmp(sFinal, sUrl) != 0) daslog_info_v("Redirected to %s", sFinal); + if(res.pMime) daslog_info_v("Content type %s", res.pMime); + + das_stdio_binary(stdout); + + /* The library closes the connection, success or not. Only the write + * failure is ours to classify, everything else is the server's fault + * as far as the exit code is concerned. */ + int64_t nTotal = 0; + int nRet = RET_OK; + switch(das_http_copyBody(&res, stdout, &nTotal, NULL, NULL)){ + case DAS_OKAY: nRet = RET_OK; break; + case DASERR_IO: nRet = RET_WRITE; break; + default: nRet = RET_SERVER; break; + } + fflush(stdout); + + DasHttpResp_freeFields(&res); + + daslog_info_v("%" PRId64 " bytes written", nTotal); + return nRet; +} + +/* ************************************************************************* */ + +/* Columns available on stdout, 0 when it is not a terminal */ +static int termWidth(void) +{ + if(!isatty(fileno(stdout))) return 0; +#ifdef _WIN32 + CONSOLE_SCREEN_BUFFER_INFO info; + if(GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info)) + return info.srWindow.Right - info.srWindow.Left + 1; +#else + struct winsize ws; + if((ioctl(fileno(stdout), TIOCGWINSZ, &ws) == 0) && (ws.ws_col > 0)) return ws.ws_col; +#endif + const char* sCols = getenv("COLUMNS"); + if(isDigits(sCols)) return atoi(sCols); + return 80; +} + +/* -s SECTIONS: comma separated section names to a bit set */ +static bool parseShow(const char* sList, unsigned* puShow) +{ + char sCopy[256]; + strcpyz(sCopy, sList, sizeof(sCopy)); + *puShow = 0; + for(char* pTok = strtok(sCopy, ","); pTok != NULL; pTok = strtok(NULL, ",")){ + das_src_sec_e sec; + if(strieq(pTok, "all")) *puShow = SHOW_ALL; + else if(strieq(pTok, "examples")||strieq(pTok, "example")||strieq(pTok, "ex")) *puShow |= SHOW_EXAMPLES; + else if(secAlias(pTok, &sec)) *puShow |= (1u << sec); + else return false; + } + return true; +} + +int main(int argc, char** argv) +{ + das_init(PROG, DASERR_DIS_EXIT, 0, DASLOG_WARN, NULL); + daslog_set_showline(DASLOG_NOTHING); /* messages here are for users, not debuggers */ + + char sAgent[64]; + snprintf(sAgent, sizeof(sAgent), PROG " (das2C %s)", das_version()); + DasCat* pCat = NULL; /* made once the arguments are in, see below */ + + const char* sCmd = NULL; + const char* sPath = NULL; + const char* sCacheArg = NULL; + const char* sStartArg = NULL; + const char* aMountArg[MAX_MOUNTS]; + size_t uMountArg = 0; + const char* sAccept = NULL; /* -a, else the two step default in pickSrc */ + bool bHelp = false; + bool bFresh = false; + bool bQuiet = false; + const char* aKeyVals[MAX_KEYS]; /* KEY=VALUE arguments, applied in order */ + size_t uKeyVals = 0; + const char* aRaw[MAX_RAW]; /* -p PARAM=VALUE arguments */ + size_t uRaw = 0; + int nExample = DASSRC_BLANK_REQ; + float rTimeout = 10.0; + + for(int i = 1; i < argc; ++i){ + const char* sArg = argv[i]; + if((strcmp(sArg, "-h") == 0)||(strcmp(sArg, "--help") == 0)){ bHelp = true; continue; } + if((strcmp(sArg, "-V") == 0)||(strcmp(sArg, "--version") == 0)){ + printf(PROG " (das2C %s)\n", das_version()); + return RET_OK; + } + if(strcmp(sArg, "-q") == 0){ daslog_setlevel(DASLOG_ERROR); bQuiet = true; continue; } + if(strcmp(sArg, "-v") == 0){ daslog_setlevel(DASLOG_INFO); g_bVerbose = true; g_uShow = SHOW_ALL; continue; } + if(strcmp(sArg, "-f") == 0){ bFresh = true; continue; } + if(strcmp(sArg, "--start") == 0) sArg = "-r"; + if(strcmp(sArg, "--mount") == 0) sArg = "-m"; + if((strcmp(sArg, "-r") == 0)||(strcmp(sArg, "-m") == 0)||(strcmp(sArg, "-s") == 0)|| + (strcmp(sArg, "-C") == 0)||(strcmp(sArg, "-p") == 0)||(strcmp(sArg, "-t") == 0)|| + (strcmp(sArg, "-l") == 0)||(strcmp(sArg, "-a") == 0)){ + if(i + 1 >= argc){ + fprintf(stderr, PROG ": option %s needs a value, use -h for help\n", sArg); + return RET_USAGE; + } + const char* sVal = argv[++i]; + switch(sArg[1]){ + case 'r': sStartArg = sVal; break; + case 'm': + if(uMountArg >= MAX_MOUNTS){ + fprintf(stderr, PROG ": at most %d mounts\n", MAX_MOUNTS); + return RET_USAGE; + } + aMountArg[uMountArg++] = sVal; + break; + case 'a': sAccept = sVal; break; + case 's': + if(!parseShow(sVal, &g_uShow)){ + fprintf(stderr, PROG ": -s takes a list from coords, data, formats, options, " + "examples, all\n"); + return RET_USAGE; + } + break; + case 'C': sCacheArg = sVal; break; + case 't': rTimeout = (float)atof(sVal); break; + case 'l':{ + int nLvl = daslog_strlevel(sVal); + daslog_setlevel(nLvl); + break; + } + case 'p':{ + const char* pEq = strchr(sVal, '='); + if((pEq == NULL)||(pEq == sVal)||(uRaw >= MAX_RAW)){ + fprintf(stderr, PROG ": -p needs PARAM=VALUE (at most %d of them)\n", MAX_RAW); + return RET_USAGE; + } + aRaw[uRaw++] = sVal; + break; + } + } + continue; + } + if(sArg[0] == '-'){ + fprintf(stderr, PROG ": unknown option %s, use -h for help\n", sArg); + return RET_USAGE; + } + + /* Positionals: command, path, then KEY=VALUE pairs. A bare number is + an example index, not a path. */ + if(sCmd == NULL){ sCmd = sArg; continue; } + if((sPath == NULL) && (strchr(sArg, '=') == NULL) && !isDigits(sArg)){ sPath = sArg; continue; } + if(uKeyVals < MAX_KEYS) aKeyVals[uKeyVals++] = sArg; + } + if(bHelp){ prnHelp(sCmd); return RET_OK; } + g_nCols = termWidth(); + + /* Credentials: saved ones are tried before the terminal prompt. The + catalog owns the manager and hands it to every node and request. */ + char sCredFile[256] = {'\0'}; + snprintf(sCredFile, sizeof(sCredFile), "%s" DAS_DSEPS CRED_FILE, das_userdir()); + DasCredMngr* pMgr = new_CredMngr(sCredFile); + + /* CredMngr_load logs an error for a missing file even though its + contract says a missing file is not an error. Most users have no + saved credentials, so look first. */ + FILE* pTest = fopen(sCredFile, "r"); + if(pTest != NULL){ + fclose(pTest); + CredMngr_load(pMgr, NULL, NULL); + } + pCat = new_DasCat(pMgr, sAgent); + + /* Environment fallbacks for -r and -m, for people who live in one test + catalog. The start is applied first so a mount can complain about a + conflicting one. */ + const char* sEnv = getenv(START_ENV); + if((sStartArg == NULL) && (sEnv != NULL) && (sEnv[0] != '\0')) sStartArg = sEnv; + sEnv = getenv(MOUNT_ENV); + if((uMountArg == 0) && (sEnv != NULL) && (sEnv[0] != '\0')) aMountArg[uMountArg++] = sEnv; + if((sStartArg != NULL) && !applyStartArg(pCat, sStartArg, false)) + return finish(pCat, RET_USAGE); + for(size_t u = 0; u < uMountArg; ++u) + if(!applyStartArg(pCat, aMountArg[u], true)) return finish(pCat, RET_USAGE); + + if(sCmd == NULL){ + fprintf(stderr, PROG ": no command given, use -h for help\n"); + return finish(pCat, RET_USAGE); + } + + if(strcmp(sCmd, "seeds") == 0){ + size_t uSeeds = 0; + const char** psSeeds = das_seed_urls(&uSeeds); + for(size_t u = 0; u < uSeeds; ++u) printf("%s\n", psSeeds[u]); + return finish(pCat, RET_OK); + } + + bool bNode = (strcmp(sCmd, "node") == 0)||(strcmp(sCmd, "ls") == 0); + bool bJson = (strcmp(sCmd, "json") == 0); + bool bUrl = (strcmp(sCmd, "url") == 0); + bool bData = (strcmp(sCmd, "data") == 0)||(strcmp(sCmd, "get") == 0); + if(!bNode && !bJson && !bUrl && !bData){ + fprintf(stderr, PROG ": unknown command '%s', use -h for help\n", sCmd); + return finish(pCat, RET_USAGE); + } + /* Mounts are consulted on the way, they are not a place to start */ + if((bJson || bUrl || bData) && (sPath == NULL) && !DasCat_hasSeed(pCat)){ + fprintf(stderr, PROG ": the %s command needs a PATH, use -h for help\n", sCmd); + return finish(pCat, RET_USAGE); + } + if(bNode && (uKeyVals > 0)) + daslog_warn("KEY=VALUE arguments are ignored by the node command"); + + /* Node cache: the session's default unless told otherwise. -f refreshes + rather than disables so that the run after a catalog change leaves the + cache current. */ + if((sCacheArg != NULL) && strieq(sCacheArg, "none")) + DasCat_setCache(pCat, NULL, 0); + else if(sCacheArg != NULL) + DasCat_setCache(pCat, sCacheArg, bFresh ? 0 : DASCAT_DEF_CACHE_SEC); + else if(bFresh) + DasCat_setCache(pCat, DasCat_cacheDir(pCat), 0); + + /* A fragment on the path is only meaningful for the json command */ + char sPathBuf[1024] = {'\0'}; + const char* sFrag = NULL; + if(sPath != NULL){ + strcpyz(sPathBuf, sPath, sizeof(sPathBuf)); + char* pHash = strchr(sPathBuf, '#'); + if(pHash != NULL){ *pHash = '\0'; sFrag = pHash + 1; } + /* "a/b/" means the same as "a/b", shell completion adds the slash */ + if(!das_isUrlOrFile(sPathBuf)){ + size_t u = strlen(sPathBuf); + while((u > 1) && (sPathBuf[u-1] == '/') && (sPathBuf[u-2] != ':')) sPathBuf[--u] = '\0'; + } + sPath = sPathBuf; + } + + DasNode* pNode = DasCat_resolve(pCat, sPath); + if(pNode == NULL){ + explainUnresolved(pCat, sPath); + return finish(pCat, RET_NODE); + } + char sShow[1200]; + nodeUri(pNode, DasCat_lastUri(pCat), sShow, sizeof(sShow)); + + int nRet = RET_OK; + + if(bJson){ + const DasJdo* pJdo = DasNode_getJdo(pNode, sFrag); + if(pJdo == NULL){ + daslog_error_v("Node has no element at '%s'", sFrag ? sFrag : ""); + nRet = RET_NODE; + } + else{ + char* sPretty = (char*)DasJdo_writePretty(pJdo, " ", "\n", NULL); + if(sPretty){ fputs(sPretty, stdout); fputs("\n", stdout); free(sPretty); } + } + return finish(pCat, nRet); + } + + if(bNode){ + if(DasNode_isCatalog(pNode)){ + /* A source container is one step from the goal. Users never type + the source's name, so by default act as if they had asked for the + default source and describe it under the container's URI. The + container listing itself only matters once more than one source + type can be fetched from, so it is verbose only for now. */ + das_node_type_e nType = DasNode_type(pNode); + bool bSrcSet = (nType == d2node_srcset)||(nType == d2node_collection); + if(!bSrcSet || g_bVerbose) listCatalog(pNode, stdout, sShow); + + if(bSrcSet){ + DasSrc* pSrc = pickSrc(pCat, pNode, sAccept); + if(pSrc == NULL){ + /* pickSrc said why; the listing shows what to name */ + if(!g_bVerbose) listCatalog(pNode, stdout, sShow); + nRet = RET_NODE; + } + else if(!g_bVerbose){ + describeSrc(pSrc, stdout, sShow); + } + else{ + /* The child's name is the tail of its path URI, after the + separator this container uses */ + const char* sSrcUri = DasSrc_path(pSrc); + const char* sSep = DasNode_pathSep(pNode); + const char* sName = sSrcUri; + if(sSep && sSep[0]){ + const char* p = strstr(sSrcUri, sSep); + while(p){ sName = p + strlen(sSep); p = strstr(sName, sSep); } + } + char sNote[256]; + snprintf(sNote, sizeof(sNote), "Default source: %s", sName); + fputc('\n', stdout); + prnPara(stdout, 0, sNote); + fputc('\n', stdout); + describeSrc(pSrc, stdout, sSrcUri); + } + del_DasSrc(pSrc); + } + } + else if(DasNode_isStreamSrc(pNode)){ + DasSrc* pSrc = new_DasSrc_node(pNode); + if(pSrc == NULL){ + nRet = RET_NODE; + } + else{ + describeSrc(pSrc, stdout, sShow); + del_DasSrc(pSrc); + } + } + else{ + const char* sType = jstr(DasNode_getJdo(pNode, NULL), D2FRAG_TYPE); + printf("%s: %s\n%-10s%s\n", sType ? sType : "Node", + DasNode_label(pNode) ? DasNode_label(pNode) : "", "URI:", sShow); + if(g_bVerbose) printf("From: %s\n", DasNode_srcUrl(pNode)); + printf("\n" PROG " has no detailed printer for this node type, try the json command.\n"); + } + return finish(pCat, nRet); + } + + /* url or data. The source outlives nothing here, but it does not need + the tree either, so the order of teardown does not matter. */ + DasSrc* pSrc = pickSrc(pCat, pNode, sAccept); + if(pSrc == NULL) return finish(pCat, RET_NODE); + + /* Example selectors among the keys: examples.N, or bare N as the shortcut, + the same pairing as data.sprt and sprt. They are pulled out first since + an example is the base the other keys modify. */ + size_t uKept = 0; + for(size_t u = 0; u < uKeyVals; ++u){ + const char* sArg = aKeyVals[u]; + const char* sNum = sArg; + if(strchr(sArg, '=') == NULL){ + const char* pDot = strchr(sArg, '.'); + if(pDot != NULL){ + size_t uLen = (size_t)(pDot - sArg); + bool bSec = ((uLen == 7) && (strncmp(sArg, "example", 7) == 0)) || + ((uLen == 8) && (strncmp(sArg, "examples", 8) == 0)); + if(bSec){ + if(!isDigits(pDot + 1)){ + daslog_error_v("Examples are numbered, there is no '%s'", sArg); + del_DasSrc(pSrc); + return finish(pCat, RET_QUERY); + } + sNum = pDot + 1; + } + } + if(isDigits(sNum)){ nExample = atoi(sNum); continue; } + } + aKeyVals[uKept++] = sArg; + } + uKeyVals = uKept; + + /* Nothing asked for at all: run the first example so there is always a + quick way to see a source work, and say so */ + if((nExample < 0) && (uKeyVals == 0) && (uRaw == 0) && (DasSrc_exampleCount(pSrc) > 0)){ + nExample = 0; + if(!bQuiet) + fprintf(stderr, PROG ": no query given, using example 0, \"%s\"\n", DasSrc_exampleLabel(pSrc, 0)); + } + + /* Order matters: the example is the base, explicit keys override it, and + -p raw parameters have the last word when the URL is built. */ + DasReq* pReq = DasSrc_newReq(pSrc, nExample); + if(pReq == NULL){ + del_DasSrc(pSrc); + return finish(pCat, RET_QUERY); + } + for(size_t u = 0; u < uKeyVals; ++u){ + char sKey[KEY_SZ]; + const char* pEq = strchr(aKeyVals[u], '='); + const char* sVal = "true"; /* bare KEY toggles a boolean on */ + if(pEq != NULL){ + size_t uLen = (size_t)(pEq - aKeyVals[u]); + if(uLen >= KEY_SZ) uLen = KEY_SZ - 1; + memcpy(sKey, aKeyVals[u], uLen); + sKey[uLen] = '\0'; + sVal = pEq + 1; + } + else{ + strcpyz(sKey, aKeyVals[u], KEY_SZ); + } + DasReq_set(pReq, sKey, sVal); /* only an empty key fails, and none is */ + } + for(size_t u = 0; u < uRaw; ++u){ + char sName[KEY_SZ]; + const char* pEq = strchr(aRaw[u], '='); /* present, main checked */ + size_t uLen = (size_t)(pEq - aRaw[u]); + if(uLen >= KEY_SZ) uLen = KEY_SZ - 1; + memcpy(sName, aRaw[u], uLen); + sName[uLen] = '\0'; + DasReq_setRaw(pReq, sName, pEq + 1); + } + + /* The library logs the first bad entry, a missing required parameter, + or a source with nothing to query, so a failed URL needs no more + words here. Too small a buffer is the one case that is ours. */ + char sUrl[URL_SZ]; + if(bUrl){ + size_t uNeed = DasStreamSrc_url(pSrc, pReq, 0, sUrl, sizeof(sUrl)); + if(uNeed == 0) nRet = RET_QUERY; + else if(uNeed >= sizeof(sUrl)){ daslog_error("Query URL too long"); nRet = RET_QUERY; } + else printf("%s\n", sUrl); + del_DasReq(pReq); + del_DasSrc(pSrc); + return finish(pCat, nRet); + } + + /* data: try each base URL in turn until one answers. A server that + answers with an error is an answer, only connection failures move on. + Base 0 is always asked for, so a source with no base URLs at all is + reported by the library like any other unusable one. */ + size_t uBases = DasStreamSrc_urlCount(pSrc); + for(size_t u = 0; ; ++u){ + size_t uNeed = DasStreamSrc_url(pSrc, pReq, u, sUrl, sizeof(sUrl)); + if(uNeed == 0){ nRet = RET_QUERY; break; } + if(uNeed >= sizeof(sUrl)){ daslog_error("Query URL too long"); nRet = RET_QUERY; break; } + nRet = streamUrl(pCat, sUrl, rTimeout); + if((nRet != RET_SERVER)||(u + 1 >= uBases)) break; + daslog_warn_v("Trying alternate server %s", DasStreamSrc_baseUrl(pSrc, u + 1)); + } + + del_DasReq(pReq); + del_DasSrc(pSrc); + finish(pCat, nRet); + das_finish(); + return nRet; +} diff --git a/utilities/validrange.h b/utilities/validrange.h index d407367..c4d1029 100644 --- a/utilities/validrange.h +++ b/utilities/validrange.h @@ -1,6 +1,4 @@ /* Copyright (C) 2026 Chris Piker - * - * Author: C. Piker, via Claude Fable 5.1 * * This file is part of das2C, the Core Das2 C Library. *